d2f816d6ff · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/models/ernie45_vl.py

+3 -3

@@ -1645,12 +1645,12 @@ class Ernie4_5_VLMoeForConditionalGeneration(

1645 for modality in modalities:

1646 if modality == "images":

1647 image_input = modalities["images"]

1648 - vision_embeddings = self._process_image_input(image_input)

1649 - multimodal_embeddings += vision_embeddings

1648 + image_embeddings = self._process_image_input(image_input)

1649 + multimodal_embeddings += tuple(image_embeddings)

1650 if modality == "videos":

1651 video_input = modalities["videos"]

1652 video_embeddings = self._process_video_input(video_input)

1653 - multimodal_embeddings += video_embeddings

1653 + multimodal_embeddings += tuple(video_embeddings)

1654

1655 return multimodal_embeddings

1656

vllm/model_executor/models/glm4_1v.py

+3 -3

@@ -1608,11 +1608,11 @@ class Glm4vForConditionalGeneration(

1608 for modality in mm_input_by_modality:

1609 multimodal_input = mm_input_by_modality[modality]

1610 if modality == "image":

1611 - vision_embeddings = self._process_image_input(multimodal_input)

1612 - multimodal_embeddings += vision_embeddings

1611 + image_embeddings = self._process_image_input(multimodal_input)

1612 + multimodal_embeddings += tuple(image_embeddings)

1613 if modality == "video":

1614 video_embeddings = self._process_video_input(multimodal_input)

1615 - multimodal_embeddings += video_embeddings

1615 + multimodal_embeddings += tuple(video_embeddings)

1616 return multimodal_embeddings

1617

1618 def forward(

vllm/model_executor/models/hyperclovax_vision.py

+3 -3

@@ -749,12 +749,12 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

749 for modality in modalities:

750 if modality == "images":

751 image_input = modalities["images"]

752 - vision_embeddings = self._process_image_input(image_input)

753 - multimodal_embeddings += vision_embeddings

752 + image_embeddings = self._process_image_input(image_input)

753 + multimodal_embeddings += tuple(image_embeddings)

754 if modality == "videos":

755 video_input = modalities["videos"]

756 video_embeddings = self._process_video_input(video_input)

757 - multimodal_embeddings += video_embeddings

757 + multimodal_embeddings += tuple(video_embeddings)

758

759 return multimodal_embeddings

760

vllm/model_executor/models/interns1.py

+3 -3

@@ -753,12 +753,12 @@ class InternS1ForConditionalGeneration(

753 for modality in modalities:

754 if modality == "images":

755 image_input = modalities["images"]

756 - vision_embeddings = self._process_vision_input(image_input)

757 - multimodal_embeddings += vision_embeddings

756 + image_embeddings = self._process_vision_input(image_input)

757 + multimodal_embeddings += tuple(image_embeddings)

758 if modality == "videos":

759 video_input = modalities["videos"]

760 video_embeddings = self._process_vision_input(video_input)

761 - multimodal_embeddings += video_embeddings

761 + multimodal_embeddings += tuple(video_embeddings)

762

763 return multimodal_embeddings

764

vllm/model_executor/models/internvl.py

+3 -3

@@ -1358,12 +1358,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)

1358 for modality in modalities:

1359 if modality == "images":

1360 image_input = modalities["images"]

1361 - vision_embeddings = self._process_vision_input(image_input)

1362 - multimodal_embeddings += vision_embeddings

1361 + image_embeddings = self._process_vision_input(image_input)

1362 + multimodal_embeddings += tuple(image_embeddings)

1363 if modality == "videos":

1364 video_input = modalities["videos"]

1365 video_embeddings = self._process_vision_input(video_input)

1366 - multimodal_embeddings += video_embeddings

1366 + multimodal_embeddings += tuple(video_embeddings)

1367

1368 return multimodal_embeddings

1369

vllm/model_executor/models/keye.py

+3 -3

@@ -1459,12 +1459,12 @@ class BaseKeyeModule(nn.Module):

1459 for modality in modalities:

1460 if modality == "images":

1461 image_input = modalities["images"]

1462 - vision_embeddings = self._process_image_input(image_input)

1463 - multimodal_embeddings += vision_embeddings

1462 + image_embeddings = self._process_image_input(image_input)

1463 + multimodal_embeddings += tuple(image_embeddings)

1464 if modality == "videos":

1465 video_input = modalities["videos"]

1466 video_embeddings = self._process_video_input(video_input)

1467 - multimodal_embeddings += video_embeddings

1467 + multimodal_embeddings += tuple(video_embeddings)

1468 return multimodal_embeddings

1469

1470 def forward(

vllm/model_executor/models/llava_onevision.py

+2 -2

@@ -881,8 +881,8 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp

881 for modality in mm_input_by_modality:

882 multimodal_input = mm_input_by_modality[modality]

883 if modality == "image":

884 - vision_embeddings = self._process_image_input(multimodal_input)

885 - multimodal_embeddings += tuple(vision_embeddings)

884 + image_embeddings = self._process_image_input(multimodal_input)

885 + multimodal_embeddings += tuple(image_embeddings)

886 if modality == "video":

887 video_embeddings = self._process_video_pixels(multimodal_input)

888 multimodal_embeddings += tuple(video_embeddings)

vllm/model_executor/models/minicpmo.py

+2 -2

@@ -762,7 +762,7 @@ class MiniCPMO(MiniCPMV2_6):

762 for modality in modalities:

763 if modality == "audios":

764 audio_input = modalities["audios"]

765 - audio_features = self._process_audio_input(audio_input)

766 - multimodal_embeddings += tuple(audio_features)

765 + audio_embeddings = self._process_audio_input(audio_input)

766 + multimodal_embeddings += tuple(audio_embeddings)

767

768 return multimodal_embeddings

vllm/model_executor/models/minicpmv.py

+4 -4

@@ -1129,12 +1129,12 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

1129 for modality in modalities:

1130 if modality == "images":

1131 image_input = modalities["images"]

1132 - image_features = self._process_vision_input(image_input)

1133 - multimodal_embeddings += tuple(image_features)

1132 + image_embeddings = self._process_vision_input(image_input)

1133 + multimodal_embeddings += tuple(image_embeddings)

1134 if modality == "videos":

1135 video_input = modalities["videos"]

1136 - video_features = self._process_vision_input(video_input)

1137 - multimodal_embeddings += tuple(video_features)

1136 + video_embeddings = self._process_vision_input(video_input)

1137 + multimodal_embeddings += tuple(video_embeddings)

1138

1139 return multimodal_embeddings

1140

vllm/model_executor/models/nano_nemotron_vl.py

+3 -3

@@ -1263,12 +1263,12 @@ class NemotronH_Nano_VL_V2(

1263 for modality in modalities:

1264 if modality == "images":

1265 image_input = modalities["images"]

1266 - vision_embeddings = self._process_image_input(image_input)

1267 - multimodal_embeddings += vision_embeddings

1266 + image_embeddings = self._process_image_input(image_input)

1267 + multimodal_embeddings += tuple(image_embeddings)

1268 if modality == "videos":

1269 video_input = modalities["videos"]

1270 video_embeddings = self._process_video_input(video_input)

1271 - multimodal_embeddings += video_embeddings

1271 + multimodal_embeddings += tuple(video_embeddings)

1272

1273 return multimodal_embeddings

1274

vllm/model_executor/models/nemotron_vl.py

+2 -2

@@ -575,8 +575,8 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor

575 for modality in modalities:

576 if modality == "images":

577 image_input = modalities["images"]

578 - vision_embeddings = self._process_image_input(image_input)

579 - multimodal_embeddings += vision_embeddings

578 + image_embeddings = self._process_image_input(image_input)

579 + multimodal_embeddings += tuple(image_embeddings)

580

581 return multimodal_embeddings

582

vllm/model_executor/models/ovis2_5.py

+3 -3

@@ -616,12 +616,12 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):

616 for modality in modalities:

617 if modality == "images":

618 image_input = modalities["images"]

619 - vision_embeddings = self._process_visual_input(image_input)

620 - multimodal_embeddings += vision_embeddings

619 + image_embeddings = self._process_visual_input(image_input)

620 + multimodal_embeddings += tuple(image_embeddings)

621 if modality == "videos":

622 video_input = modalities["videos"]

623 video_embeddings = self._process_visual_input(video_input)

624 - multimodal_embeddings += video_embeddings

624 + multimodal_embeddings += tuple(video_embeddings)

625

626 return multimodal_embeddings

627

vllm/model_executor/models/phi4_multimodal.py

+2 -2

@@ -1430,8 +1430,8 @@ class Phi4MultimodalForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

1430 if modality == "images":

1431 audio_projection_mode = "vision"

1432 image_input = modalities["images"]

1433 - vision_embeddings = self._process_image_input(image_input)

1434 - multimodal_embeddings += tuple(vision_embeddings)

1433 + image_embeddings = self._process_image_input(image_input)

1434 + multimodal_embeddings += tuple(image_embeddings)

1435 if modality == "audios":

1436 audio_input = modalities["audios"]

1437 audio_embeddings = self._process_audio_input(

vllm/model_executor/models/phi4mm.py

+2 -2

@@ -1248,8 +1248,8 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

1248 if modality == "images":

1249 audio_projection_mode = "vision"

1250 image_input = modalities["images"]

1251 - vision_embeddings = self._process_image_input(image_input)

1252 - multimodal_embeddings += tuple(vision_embeddings)

1251 + image_embeddings = self._process_image_input(image_input)

1252 + multimodal_embeddings += tuple(image_embeddings)

1253 if modality == "audios":

1254 audio_input = modalities["audios"]

1255 audio_embeddings = self._process_audio_input(

vllm/model_executor/models/qwen2_5_omni_thinker.py

+4 -4

@@ -1210,14 +1210,14 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

1210 for modality in mm_input_by_modality:

1211 multimodal_input = mm_input_by_modality[modality]

1212 if modality == "image":

1213 - vision_embeddings = self._process_image_input(multimodal_input)

1214 - multimodal_embeddings += vision_embeddings

1213 + image_embeddings = self._process_image_input(multimodal_input)

1214 + multimodal_embeddings += tuple(image_embeddings)

1215 if modality == "video":

1216 video_embeddings = self._process_video_input(multimodal_input)

1217 - multimodal_embeddings += video_embeddings

1217 + multimodal_embeddings += tuple(video_embeddings)

1218 if modality == "audio":

1219 audio_embeddings = self._process_audio_input(multimodal_input)

1220 - multimodal_embeddings += audio_embeddings

1220 + multimodal_embeddings += tuple(audio_embeddings)

1221 return multimodal_embeddings

1222

1223 # TODO (ywang96): support overlapping modality embeddings so that

vllm/model_executor/models/qwen2_5_vl.py

+5 -5

@@ -1586,19 +1586,19 @@ class Qwen2_5_VLForConditionalGeneration(

1586 for modality in mm_input_by_modality:

1587 multimodal_input = mm_input_by_modality[modality]

1588 if modality == "image":

1589 - vision_embeddings = self._process_image_input(multimodal_input)

1589 + image_embeddings = self._process_image_input(multimodal_input)

1590 if self.is_multimodal_pruning_enabled:

1591 - vision_embeddings = self._postprocess_image_embeds_evs(

1592 - vision_embeddings, multimodal_input

1591 + image_embeddings = self._postprocess_image_embeds_evs(

1592 + image_embeddings, multimodal_input

1593 )

1594 - multimodal_embeddings += vision_embeddings

1594 + multimodal_embeddings += tuple(image_embeddings)

1595 if modality == "video":

1596 video_embeddings = self._process_video_input(multimodal_input)

1597 if self.is_multimodal_pruning_enabled:

1598 video_embeddings = self._postprocess_video_embeds_evs(

1599 video_embeddings, multimodal_input

1600 )

1601 - multimodal_embeddings += video_embeddings

1601 + multimodal_embeddings += tuple(video_embeddings)

1602 return multimodal_embeddings

1603

1604 def forward(

vllm/model_executor/models/qwen2_vl.py

+3 -3

@@ -1561,12 +1561,12 @@ class Qwen2VLForConditionalGeneration(

1561 for modality in modalities:

1562 if modality == "images":

1563 image_input = modalities["images"]

1564 - vision_embeddings = self._process_image_input(image_input)

1565 - multimodal_embeddings += vision_embeddings

1564 + image_embeddings = self._process_image_input(image_input)

1565 + multimodal_embeddings += tuple(image_embeddings)

1566 if modality == "videos":

1567 video_input = modalities["videos"]

1568 video_embeddings = self._process_video_input(video_input)

1569 - multimodal_embeddings += video_embeddings

1569 + multimodal_embeddings += tuple(video_embeddings)

1570

1571 return multimodal_embeddings

1572

vllm/model_executor/models/qwen3_omni_moe_thinker.py

+4 -4

@@ -1260,14 +1260,14 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1260 for modality in mm_input_by_modality:

1261 multimodal_input = mm_input_by_modality[modality]

1262 if modality == "image":

1263 - vision_embeddings = self._process_image_input(multimodal_input)

1264 - multimodal_embeddings += vision_embeddings

1263 + image_embeddings = self._process_image_input(multimodal_input)

1264 + multimodal_embeddings += tuple(image_embeddings)

1265 if modality == "video":

1266 video_embeddings = self._process_video_input(multimodal_input)

1267 - multimodal_embeddings += video_embeddings

1267 + multimodal_embeddings += tuple(video_embeddings)

1268 if modality == "audio":

1269 audio_embeddings = self._process_audio_input(multimodal_input)

1270 - multimodal_embeddings += audio_embeddings

1270 + multimodal_embeddings += tuple(audio_embeddings)

1271 return multimodal_embeddings

1272

1273 def get_input_embeddings(

vllm/model_executor/models/qwen3_vl.py

+3 -3

@@ -1601,11 +1601,11 @@ class Qwen3VLForConditionalGeneration(

1601 for modality in mm_input_by_modality:

1602 multimodal_input = mm_input_by_modality[modality]

1603 if modality == "image":

1604 - vision_embeddings = self._process_image_input(multimodal_input)

1605 - multimodal_embeddings += vision_embeddings

1604 + image_embeddings = self._process_image_input(multimodal_input)

1605 + multimodal_embeddings += tuple(image_embeddings)

1606 if modality == "video":

1607 video_embeddings = self._process_video_input(multimodal_input)

1608 - multimodal_embeddings += video_embeddings

1608 + multimodal_embeddings += tuple(video_embeddings)

1609 return multimodal_embeddings

1610

1611 def _compute_deepstack_embeds(

[Bugfix] Standardize merging multimodal embeddings (#26771)