From caee26e3c3b94df804e625984009b691a42283d5 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 30 Sep 2026 19:29:18 -0700 Subject: [PATCH] llm: patch 04b v2 - plan-time signal is the spec-config METHOD use_aux_hidden_state_outputs lives on the gpu_model_runner instance (gpu_model_runner.py:623/678), not on the spec config, so the first registry check never fired and the reserve stayed at 13,600 B/token. Switch to method in {dflash, dspark, extract_hidden_states}. --- .../glm53-dflash-04b-kv-registry-dflash.patch | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/patches/glm53-dflash-04b-kv-registry-dflash.patch b/scripts/patches/glm53-dflash-04b-kv-registry-dflash.patch index 4229db2..8941bc1 100644 --- a/scripts/patches/glm53-dflash-04b-kv-registry-dflash.patch +++ b/scripts/patches/glm53-dflash-04b-kv-registry-dflash.patch @@ -1,6 +1,6 @@ --- a/vllm/model_executor/offloader/expert_activation_registry.py +++ b/vllm/model_executor/offloader/expert_activation_registry.py -@@ -316,7 +316,19 @@ +@@ -316,7 +316,27 @@ def _glm53_kv_bytes_per_token(cfg: "VllmConfig") -> int: @@ -13,8 +13,16 @@ + # cannot absorb, so the reserve must include them. The drafter is a 5-layer + # qwen3 (8 kv heads x 128 x K+V x bf16 = 4096 B/token); the aux count is + # the dflash target_layer_ids count (5). ++ # NOTE: use_aux_hidden_state_outputs lives on the gpu_model_runner ++ # instance (gpu_model_runner.py:623/678), not on the spec config -- it is ++ # not set yet when the expert plan runs. The spec-config METHOD is the ++ # plan-time signal for the same fact. + sc = getattr(cfg, "speculative_config", None) -+ if sc is not None and getattr(sc, "use_aux_hidden_state_outputs", False): ++ if sc is not None and getattr(sc, "method", "") in ( ++ "dflash", ++ "dspark", ++ "extract_hidden_states", ++ ): + hidden = int(cfg.model_config.hf_config.hidden_size) + per_tok += 5 * hidden * 2 + 4096 + return per_tok