--- a/vllm/model_executor/offloader/expert_activation_registry.py +++ b/vllm/model_executor/offloader/expert_activation_registry.py @@ -316,7 +316,27 @@ def _glm53_kv_bytes_per_token(cfg: "VllmConfig") -> int: - return 13600 + per_tok = 13600 + # FORK-LOCAL (patches/glm53-dflash-04-kv-groups-draft): with a dflash/dspark + # drafter, the KV spec set gains the drafter's attention layers and the aux + # hidden-state cache (one bf16 hidden vector per target aux layer per + # token). Both are full-length per-token costs the aliased glm5 pool + # cannot absorb, so the reserve must include them. The drafter is a 5-layer + # qwen3 (8 kv heads x 128 x K+V x bf16 = 4096 B/token); the aux count is + # the dflash target_layer_ids count (5). + # NOTE: use_aux_hidden_state_outputs lives on the gpu_model_runner + # instance (gpu_model_runner.py:623/678), not on the spec config -- it is + # not set yet when the expert plan runs. The spec-config METHOD is the + # plan-time signal for the same fact. + sc = getattr(cfg, "speculative_config", None) + if sc is not None and getattr(sc, "method", "") in ( + "dflash", + "dspark", + "extract_hidden_states", + ): + hidden = int(cfg.model_config.hf_config.hidden_size) + per_tok += 5 * hidden * 2 + 4096 + return per_tok for _arch in ("Glm5NextForConditionalGeneration", "Glm5NextForCausalLM"):