Add MTP support for Nemotron VL models (#53121)
Signed-off-by: Daniel Serebrenik <daserebrenik@nvidia.com>
danisereb committed
Aug 24, 2026 at 03:59 UTC
a7195188a4b45dec40030467ec6b69b4f1283c8e
2 files changed
+10
-2
vllm/model_executor/models/nemotron_h_mtp.py
+6
-2
@@ -216,7 +216,7 @@ class NemotronHMultiTokenPredictor(nn.Module):
216
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
217
super().__init__()
218
219
- config = vllm_config.model_config.hf_config
219
+ config = vllm_config.model_config.hf_config.get_text_config()
220
221
self.config = config
222
self.vocab_size = config.vocab_size
@@ -322,7 +322,7 @@ class NemotronHMTP(nn.Module, SupportsPP):
322
323
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
324
super().__init__()
325
- config = vllm_config.model_config.hf_config
325
+ config = vllm_config.model_config.hf_config.get_text_config()
326
self.vllm_config = vllm_config
327
self.config = config
328
self.quant_config = vllm_config.quant_config
@@ -414,6 +414,10 @@ class NemotronHMTP(nn.Module, SupportsPP):
414
loaded_params: set[str] = set()
415
416
for name, loaded_weight in weights:
417
+ # MTP weights are nested in "language_model."
418
+ # in Multimodal Nemotron-H checkpoints.
419
+ name = name.removeprefix("language_model.")
420
+
421
# Only process MTP weights - skip all non-MTP weights
422
if not name.startswith("mtp.") and "embeddings" not in name:
423
continue
vllm/v1/spec_decode/llm_base_proposer.py
+4
@@ -1403,6 +1403,10 @@ class SpecDecodeBaseProposer:
1403
self.model.config.image_token_index = (
1404
target_model.config.media_placeholder_token_id
1405
)
1406
+ elif self.get_model_name(target_model) == "NemotronH_Nano_VL_V2":
1407
+ self.model.config.image_token_index = (
1408
+ target_model.config.img_context_token_id
1409
+ )
1410
else:
1411
self.model.config.image_token_index = (
1412
target_model.config.image_token_index