patches: glm53-dflash-01 v2 -- satisfy SupportsEagle3 by inheriting the Protocol
v1 hand-wrote set_aux_hidden_state_layers/get_eagle3_default_aux_hidden_state_layers and the boot reproduced the identical 'Model does not support EAGLE3 interface'. Reason: supports_eagle3() is an isinstance check against a runtime-checkable Protocol whose members include a required ClassVar flag (interfaces.py:1663 supports_eagle3: ClassVar[Literal[True]] = True), so methods alone do not satisfy it. Inheriting SupportsEagle3 -- the idiom every model in this image uses, including the fork's own DFlashLagunaForCausalLM -- supplies the flag, SupportsEagleBase's has_own_lm_head/has_own_embed_tokens, and working default implementations of both methods; the hand-written methods are deleted. base 008eeaa2... -> patched 5eea89c5..., apply-verified against the pinned image. Runbook: corrected method name, recorded the Protocol trap as the generalizable lesson for future spec-decode ports.
This commit is contained in:
1 parent
ce280e7295
commit
9c5c2494da
2 files changed
+63
-45
No files matched your search
@@ -17,15 +17,20 @@
|
||||
# `glm5next` implements neither half of the contract.
|
||||
#
|
||||
# THE CONTRACT (read out of this image, not assumed):
|
||||
# * interfaces.py:1694 -- `holder = getattr(parent_ref, "model", parent_ref)` then
|
||||
# `assert isinstance(holder, EagleModelMixin)` then
|
||||
# `holder._set_aux_hidden_state_layers(layers)`. The INNER trunk must inherit
|
||||
# EagleModelMixin, which supplies `aux_hidden_state_layers` and the slot helpers.
|
||||
# * interfaces.py:1712-1719 -- the default layer set is (2, L // 2, L - 3) and needs
|
||||
# the holder to expose `layers`.
|
||||
# * interfaces.py:1663 -- `supports_eagle3: ClassVar[Literal[True]] = True`. THE FLAG
|
||||
# IS A REQUIRED MEMBER FOR `isinstance`: satisfying the Protocol with methods alone
|
||||
# still raises the same RuntimeError -- v1 of this patch did exactly that and the boot
|
||||
# failed identically. INHERIT the Protocol instead. That is the idiom in this image
|
||||
# (LagunaForCausalLM, MiMoV2FlashForCausalLM and DFlashLagunaForCausalLM all list
|
||||
# SupportsEagle3 among their bases) and it brings the flag, SupportsEagleBase's
|
||||
# has_own_lm_head/has_own_embed_tokens, AND working non-stub defaults for both methods.
|
||||
# * interfaces.py:1673-1719 -- the inherited defaults unwrap
|
||||
# `holder = getattr(parent_ref, "model", parent_ref)`, assert
|
||||
# `isinstance(holder, EagleModelMixin)`, set `holder._set_aux_hidden_state_layers`, and
|
||||
# default to (2, L // 2, L - 3). So the INNER trunk must inherit EagleModelMixin,
|
||||
# which supplies `aux_hidden_state_layers` and the slot helpers.
|
||||
# * eagle3_utils.py:26-31 -- layers come from the spec config when it carries them
|
||||
# (this drafter ships target_layer_ids [5, 14, 24, 33, 42], mapped to trunk
|
||||
# indices), else from get_eagle3_default_aux_hidden_state_layers().
|
||||
# (this drafter ships target_layer_ids [5, 14, 24, 33, 42]), else from the default.
|
||||
# * deepseek_v2.py:1551-1585 and v1/worker/gpu/cudagraph_utils.py:621 -- the trunk
|
||||
# appends one tensor per aux layer while looping and returns the TUPLE
|
||||
# (hidden_states, aux_hidden_states) only when that list is non-empty.
|
||||
@@ -35,9 +40,7 @@
|
||||
# carries the already-summed single-stream state -- its non-mHC branch re-splits
|
||||
# `residual = hidden_states` inside the layer -- so this is the equivalent of
|
||||
# deepseek_v2's `hidden_states + residual`); the conditional tuple return; and
|
||||
# set_aux_hidden_state_layers / get_eagle3_default_aux_hidden_state_layers on
|
||||
# Glm5NextForCausalLM, delegating to the inner trunk (the same two-liner shape
|
||||
# kimi_k25.py:466 and pixtral.py:503 use).
|
||||
# SupportsEagle3 among Glm5NextForCausalLM's bases.
|
||||
#
|
||||
# HONEST SCOPE: this makes the interface PRESENT, which is exactly what the engine
|
||||
# checks -- it does not make drafts GOOD, and per the measured profile it is not a
|
||||
@@ -47,23 +50,26 @@
|
||||
#
|
||||
# TARGET: vllm/models/glm5next/common/model.py (inside the pinned image)
|
||||
# base sha256 008eeaa2a07e091465319f86900b5a321d921238723ccbe22e6b9e971e1d2b28
|
||||
# patched sha256 4ff6ff8ddf8af3d74bef699f98fc95a99ee47586f6e785248e7dc4b683b72d7a
|
||||
# patched sha256 5eea89c51d4ab37ef823526286b5103e39996802ca08aa1934f6373fc4728062
|
||||
#
|
||||
# APPLY: `patch -p1` from the vllm package root, or bind-mount the patched file over
|
||||
# site-packages (which is what a scratch-launcher DFLASH_PATCH=1 does).
|
||||
--- a/vllm/models/glm5next/common/model.py
|
||||
+++ b/vllm/models/glm5next/common/model.py
|
||||
@@ -95,6 +95,9 @@
|
||||
@@ -95,6 +95,12 @@
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
+from vllm.model_executor.models.interfaces import EagleModelMixin
|
||||
+from vllm.model_executor.models.interfaces import (
|
||||
+ EagleModelMixin,
|
||||
+ SupportsEagle3,
|
||||
+)
|
||||
+
|
||||
+
|
||||
class Glm5NextMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -623,7 +626,7 @@
|
||||
@@ -623,7 +629,7 @@
|
||||
)
|
||||
|
||||
|
||||
@@ -72,7 +78,7 @@
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -730,7 +733,14 @@
|
||||
@@ -730,7 +736,14 @@
|
||||
if self.is_sequence_parallel:
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
|
||||
@@ -88,7 +94,7 @@
|
||||
hidden_states, residual, post, comb = layer(
|
||||
positions, hidden_states, residual, post, comb
|
||||
)
|
||||
@@ -749,6 +759,8 @@
|
||||
@@ -749,6 +762,8 @@
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
|
||||
hidden_states = self.norm(hidden_states)
|
||||
@@ -97,21 +103,17 @@
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -963,6 +975,17 @@
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
@@ -935,7 +950,12 @@
|
||||
|
||||
+ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
+ # EAGLE-3 / DFlash2 interface (interfaces.py asserts the holder inherits
|
||||
+ # EagleModelMixin, which supplies the slot plumbing behind this attribute).
|
||||
+ self.model.aux_hidden_state_layers = layers
|
||||
+
|
||||
+ def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]:
|
||||
+ # Only used when the spec config carries no explicit aux layers; the
|
||||
+ # DFlash2 drafters this fleet runs ship target_layer_ids instead.
|
||||
+ num_layers = len(self.model.layers)
|
||||
+ return (2, num_layers // 2, num_layers - 3)
|
||||
+
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
|
||||
class Glm5NextForCausalLM(
|
||||
- nn.Module, HasInnerState, SupportsPP, MixtureOfExperts, IsHybrid
|
||||
+ nn.Module,
|
||||
+ HasInnerState,
|
||||
+ SupportsPP,
|
||||
+ MixtureOfExperts,
|
||||
+ IsHybrid,
|
||||
+ SupportsEagle3,
|
||||
):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
Reference in new issue
Block a user