[Nano-Nemotron] Fix Nano-Nemotron precomputed multimodal embeddings (#56398)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Taneem Ibrahim committed
Sep 12, 2026 at 08:07 UTC
a0914ab7d07b9a788db15b499edf3b5a280e7a14
2 files changed
+23
-4
tests/models/multimodal/test_nano_nemotron_vl.py
+12
@@ -5,6 +5,7 @@ from unittest.mock import patch
5
6
import numpy as np
7
import pytest
8
+import torch
9
10
from vllm import envs
11
from vllm.model_executor.models.nano_nemotron_vl import (
@@ -14,6 +15,17 @@ from vllm.model_executor.models.nano_nemotron_vl import (
15
from vllm.multimodal.parse import MultiModalDataItems, VideoProcessorItems
16
17
18
+@pytest.mark.parametrize("input_key", ["image_embeds", "video_embeds"])
19
+def test_precomputed_multimodal_embeddings(input_key: str):
20
+ model = object.__new__(NemotronH_Nano_VL_V2)
21
+ embeds = torch.randn(2, 4, 8)
22
+
23
+ outputs = model.embed_multimodal(**{input_key: embeds})
24
+
25
+ assert len(outputs) == len(embeds)
26
+ assert all(torch.equal(output, embed) for output, embed in zip(outputs, embeds))
27
+
28
+
29
class _TextOnlyMultiModalConfig:
30
def get_limit_per_prompt(self, modality: str) -> int:
31
return 0
vllm/model_executor/models/nano_nemotron_vl.py
+11
-4
@@ -1103,7 +1103,8 @@ class NemotronH_Nano_VL_V2(
1103
def _parse_and_validate_image_input(
1104
self, **kwargs: object
1105
) -> NanoNemotronVLImageInputs | None:
1106
- if image_embeds := kwargs.pop("image_embeds", None):
1106
+ image_embeds = kwargs.pop("image_embeds", None)
1107
+ if image_embeds is not None:
1108
return NanoNemotronVLImageEmbeddingInputs(
1109
type="image_embeds",
1110
data=image_embeds,
@@ -1341,7 +1342,7 @@ class NemotronH_Nano_VL_V2(
1342
1343
def _parse_and_validate_video_input(
1344
self, **kwargs: object
1344
- ) -> NanoNemotronVLVideoPixelInputs | None:
1345
+ ) -> NanoNemotronVLVideoInputs | None:
1346
pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
1347
video_num_patches = kwargs.pop("video_num_patches", None)
1348
video_embeds = kwargs.pop("video_embeds", None)
@@ -1408,7 +1409,10 @@ class NemotronH_Nano_VL_V2(
1409
and "images" not in modalities
1410
):
1411
modalities["images"] = self._parse_and_validate_image_input(**kwargs)
1411
- if input_key in ("pixel_values_flat_video",) and "videos" not in modalities:
1412
+ if (
1413
+ input_key in ("pixel_values_flat_video", "video_embeds")
1414
+ and "videos" not in modalities
1415
+ ):
1416
modalities["videos"] = self._parse_and_validate_video_input(**kwargs)
1417
if (
1418
input_key
@@ -1450,7 +1454,10 @@ class NemotronH_Nano_VL_V2(
1454
multimodal_embeddings += tuple(image_embeddings)
1455
if modality == "videos":
1456
video_input = modalities["videos"]
1453
- video_embeddings = self._process_video_input(video_input)
1457
+ if video_input["type"] == "video_embeds":
1458
+ video_embeddings = video_input["data"]
1459
+ else:
1460
+ video_embeddings = self._process_video_input(video_input)
1461
multimodal_embeddings += tuple(video_embeddings)
1462
if modality == "audios":
1463
audio_input = modalities["audios"]