diff --git a/scripts/patches/glm53-dflash-01-aux-hidden-states.patch b/scripts/patches/glm53-dflash-01-aux-hidden-states.patch index 21f84fc..5308233 100644 --- a/scripts/patches/glm53-dflash-01-aux-hidden-states.patch +++ b/scripts/patches/glm53-dflash-01-aux-hidden-states.patch @@ -17,15 +17,20 @@ # `glm5next` implements neither half of the contract. # # THE CONTRACT (read out of this image, not assumed): -# * interfaces.py:1694 -- `holder = getattr(parent_ref, "model", parent_ref)` then -# `assert isinstance(holder, EagleModelMixin)` then -# `holder._set_aux_hidden_state_layers(layers)`. The INNER trunk must inherit -# EagleModelMixin, which supplies `aux_hidden_state_layers` and the slot helpers. -# * interfaces.py:1712-1719 -- the default layer set is (2, L // 2, L - 3) and needs -# the holder to expose `layers`. +# * interfaces.py:1663 -- `supports_eagle3: ClassVar[Literal[True]] = True`. THE FLAG +# IS A REQUIRED MEMBER FOR `isinstance`: satisfying the Protocol with methods alone +# still raises the same RuntimeError -- v1 of this patch did exactly that and the boot +# failed identically. INHERIT the Protocol instead. That is the idiom in this image +# (LagunaForCausalLM, MiMoV2FlashForCausalLM and DFlashLagunaForCausalLM all list +# SupportsEagle3 among their bases) and it brings the flag, SupportsEagleBase's +# has_own_lm_head/has_own_embed_tokens, AND working non-stub defaults for both methods. +# * interfaces.py:1673-1719 -- the inherited defaults unwrap +# `holder = getattr(parent_ref, "model", parent_ref)`, assert +# `isinstance(holder, EagleModelMixin)`, set `holder._set_aux_hidden_state_layers`, and +# default to (2, L // 2, L - 3). So the INNER trunk must inherit EagleModelMixin, +# which supplies `aux_hidden_state_layers` and the slot helpers. # * eagle3_utils.py:26-31 -- layers come from the spec config when it carries them -# (this drafter ships target_layer_ids [5, 14, 24, 33, 42], mapped to trunk -# indices), else from get_eagle3_default_aux_hidden_state_layers(). +# (this drafter ships target_layer_ids [5, 14, 24, 33, 42]), else from the default. # * deepseek_v2.py:1551-1585 and v1/worker/gpu/cudagraph_utils.py:621 -- the trunk # appends one tensor per aux layer while looping and returns the TUPLE # (hidden_states, aux_hidden_states) only when that list is non-empty. @@ -35,9 +40,7 @@ # carries the already-summed single-stream state -- its non-mHC branch re-splits # `residual = hidden_states` inside the layer -- so this is the equivalent of # deepseek_v2's `hidden_states + residual`); the conditional tuple return; and -# set_aux_hidden_state_layers / get_eagle3_default_aux_hidden_state_layers on -# Glm5NextForCausalLM, delegating to the inner trunk (the same two-liner shape -# kimi_k25.py:466 and pixtral.py:503 use). +# SupportsEagle3 among Glm5NextForCausalLM's bases. # # HONEST SCOPE: this makes the interface PRESENT, which is exactly what the engine # checks -- it does not make drafts GOOD, and per the measured profile it is not a @@ -47,23 +50,26 @@ # # TARGET: vllm/models/glm5next/common/model.py (inside the pinned image) # base sha256 008eeaa2a07e091465319f86900b5a321d921238723ccbe22e6b9e971e1d2b28 -# patched sha256 4ff6ff8ddf8af3d74bef699f98fc95a99ee47586f6e785248e7dc4b683b72d7a +# patched sha256 5eea89c51d4ab37ef823526286b5103e39996802ca08aa1934f6373fc4728062 # # APPLY: `patch -p1` from the vllm package root, or bind-mount the patched file over # site-packages (which is what a scratch-launcher DFLASH_PATCH=1 does). --- a/vllm/models/glm5next/common/model.py +++ b/vllm/models/glm5next/common/model.py -@@ -95,6 +95,9 @@ +@@ -95,6 +95,12 @@ logger = init_logger(__name__) -+from vllm.model_executor.models.interfaces import EagleModelMixin ++from vllm.model_executor.models.interfaces import ( ++ EagleModelMixin, ++ SupportsEagle3, ++) + + class Glm5NextMLP(nn.Module): def __init__( self, -@@ -623,7 +626,7 @@ +@@ -623,7 +629,7 @@ ) @@ -72,7 +78,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() -@@ -730,7 +733,14 @@ +@@ -730,7 +736,14 @@ if self.is_sequence_parallel: hidden_states = sp_shard(hidden_states) @@ -88,7 +94,7 @@ hidden_states, residual, post, comb = layer( positions, hidden_states, residual, post, comb ) -@@ -749,6 +759,8 @@ +@@ -749,6 +762,8 @@ hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] hidden_states = self.norm(hidden_states) @@ -97,21 +103,17 @@ return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: -@@ -963,6 +975,17 @@ - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.embed_input_ids(input_ids) +@@ -935,7 +950,12 @@ -+ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: -+ # EAGLE-3 / DFlash2 interface (interfaces.py asserts the holder inherits -+ # EagleModelMixin, which supplies the slot plumbing behind this attribute). -+ self.model.aux_hidden_state_layers = layers -+ -+ def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]: -+ # Only used when the spec config carries no explicit aux layers; the -+ # DFlash2 drafters this fleet runs ship target_layer_ids instead. -+ num_layers = len(self.model.layers) -+ return (2, num_layers // 2, num_layers - 3) -+ - def forward( - self, - input_ids: torch.Tensor | None, + + class Glm5NextForCausalLM( +- nn.Module, HasInnerState, SupportsPP, MixtureOfExperts, IsHybrid ++ nn.Module, ++ HasInnerState, ++ SupportsPP, ++ MixtureOfExperts, ++ IsHybrid, ++ SupportsEagle3, + ): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() diff --git a/serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md b/serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md index 6f43628..2b0da93 100644 --- a/serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md +++ b/serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md @@ -299,7 +299,7 @@ The fork already maps that architecture: and the spec method is `"dflash"` (`config/speculative.py:67`). So the port is **target-side only**: the GLM trunk must expose auxiliary hidden states at those five layer indices, the same interface `deepseek_v2.py` (also MLA+MoE) implements — -`set_aux_hidden_state_layers()` / `get_eagle3_aux_hidden_state_layers()` plus +`set_aux_hidden_state_layers()` / `get_eagle3_default_aux_hidden_state_layers()` plus capture at `idx in self.aux_hidden_state_layers` in the layer loop. `glm5next` implements neither. @@ -316,7 +316,7 @@ RuntimeError: Model does not support EAGLE3 interface So nothing about the drafter, the method name, or the payload is wrong — the GLM trunk simply does not implement the aux-hidden-state contract the dflash runner requires. The reference is in this same image, twice: `deepseek_v2.py` -(`set_aux_hidden_state_layers` / `get_eagle3_aux_hidden_state_layers` at :1948-1951, +(`set_aux_hidden_state_layers` / `get_eagle3_default_aux_hidden_state_layers` at :1948-1951, capture at :1554 and :1579) and the fork's own `qwen4_exp` trunk. The contract is small: @@ -327,7 +327,7 @@ small: rank holds fewer tokens than `positions`); - `Glm5NextModel.forward` returns `(hidden_states, aux_hidden_states)` when the list is non-empty; -- `get_eagle3_aux_hidden_state_layers()` supplies a default triple when the drafter +- `get_eagle3_default_aux_hidden_state_layers()` supplies a default triple when the drafter config omits `target_layer_ids` (this drafter provides `[5, 14, 24, 33, 42]`, so the runner derives trunk indices `[6, 15, 25, 34, 43]`). @@ -348,14 +348,30 @@ flat decode with a prefill regression like MTP's until the pointwise mass is fus **The port exists now: `scripts/patches/glm53-dflash-01-aux-hidden-states.patch`** (WIP, independent of the compile changelist — it patches the same file as compile-01, so -compile-01 goes first if both are applied). It implements exactly the contract read out -of this image: `EagleModelMixin` on `Glm5NextModel`, capture of the single-stream -`hidden_states` ENTERING each requested layer in the loop, the conditional -`(hidden_states, aux_hidden_states)` tuple return, and -`set_aux_hidden_state_layers` / `get_eagle3_default_aux_hidden_state_layers` on -`Glm5NextForCausalLM` (delegation shape from `kimi_k25.py:466`, `pixtral.py:503`; -capture shape from `deepseek_v2.py:1551-1585`; tuple consumption at -`v1/worker/gpu/cudagraph_utils.py:621`). +compile-01 goes first if both are applied; base `008eeaa2…` → patched `5eea89c5…`). It +implements exactly the contract read out of this image: `EagleModelMixin` on +`Glm5NextModel`, capture of the single-stream `hidden_states` ENTERING each requested +layer in the loop, the conditional `(hidden_states, aux_hidden_states)` tuple return, and +`SupportsEagle3` among `Glm5NextForCausalLM`'s bases (capture shape from +`deepseek_v2.py:1551-1585`; tuple consumption at `v1/worker/gpu/cudagraph_utils.py:621`). + +**The first version of that patch was wrong in an instructive way, and it is the trap to +avoid on any future spec-decode port: `supports_eagle3` is a Protocol `isinstance` +check, not duck typing.** It hand-wrote `set_aux_hidden_state_layers` and +`get_eagle3_default_aux_hidden_state_layers` on the outer wrapper — the two methods the +traceback's call site reads — and the boot reproduced the *identical* +`RuntimeError: Model does not support EAGLE3 interface`. The reason is +`interfaces.py:1663`: `SupportsEagle3` also declares +`supports_eagle3: ClassVar[Literal[True]] = True`, and a runtime-checkable Protocol +requires every member to be *present* on the instance. **Inherit the Protocol** — the +idiom every model in the image uses (`LagunaForCausalLM`, `MiMoV2FlashForCausalLM`, +`DFlashLagunaForCausalLM` all list `SupportsEagle3` in their bases) — which supplies the +flag, `SupportsEagleBase`'s `has_own_lm_head` / `has_own_embed_tokens`, *and* working +default implementations of both methods (they unwrap +`holder = getattr(parent_ref, "model", parent_ref)`, assert it is an `EagleModelMixin`, +and default to `(2, L // 2, L - 3)`). The lesson generalizes: when a runtime error names +an interface, read the interface's declaration first — the methods a call site happens to +invoke are not necessarily the whole contract. **Retired diagnostic (do not resurrect as-is): the `/probe` startup hook.** An `op_probe.py` + `sitecustomize.py` pair was mounted to wrap vLLM module forwards and