A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 14:00 UTC
[Transformers backend] Ignore MTP weights when num_nextn_predict_layers=0 (#34888)
Signed-off-by: SteadfastAsArt <695488173@qq.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
SteadfastAsArt committed
Feb 28, 2026 at 03:39 UTC
2decec9856033347f3129f1d1b2ec015e1ad88ea
2 files changed
+15
-2
vllm/model_executor/models/transformers/base.py
+13
-1
@@ -300,14 +300,26 @@ class Base(
300
for child_name, child_module in module.named_children():
301
new_module = child_module
302
qual_name = maybe_prefix(prefix, child_name)
303
- # Populate Eagle3 attrs
303
if (
304
isinstance(module, nn.ModuleList)
305
and len(module) == self.text_config.num_hidden_layers
306
):
307
+ # Populate Eagle3 attrs
308
self._target_class = type(child_module)
309
layer_name = qual_name.removeprefix("model.")
310
self._layer_names[int(child_name)] = layer_name
311
+ # MTP weights should not be loaded into the base model
312
+ num_hidden_layers = self.text_config.num_hidden_layers
313
+ names = (
314
+ "n_predict", # Override from SpeculativeConfig
315
+ "num_nextn_predict_layers", # Most models
316
+ "mtp_num_hidden_layers", # Qwen 3.5
317
+ )
318
+ n_predict = getattr_iter(self.text_config, names, 0)
319
+ for i in range(num_hidden_layers, num_hidden_layers + n_predict):
320
+ mtp_prefix = f"{prefix}.{i}."
321
+ if mtp_prefix not in self.ignore_unexpected_prefixes:
322
+ self.ignore_unexpected_prefixes.append(mtp_prefix)
323
# Replace modules as needed
324
if isinstance(child_module, nn.Linear):
325
generator = (p for p in tp_plan if re.match(p, qual_name))
vllm/model_executor/models/utils.py
+2
-1
@@ -311,8 +311,9 @@ class AutoWeightsLoader:
311
312
continue
313
314
+ named_parameters = module.named_parameters(recurse=True)
315
desc_param_keys = {
315
- base_prefix + k for k, _ in module.named_parameters(recurse=True)
316
+ maybe_prefix(base_prefix, k) for k, _ in named_parameters
317
}
318
msg = (
319
f"There is no module or parameter named {prefix!r} "