@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 16366ee8bb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix][VLM] Fix mixed-modality inference backward compatibility for V0 (#12313)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Jan 22, 2025 at 05:06 UTC 16366ee8bbdc30aad9776b74121cfc4d8f8c897d
2 files changed +92 -28
vllm/model_executor/models/llava_onevision.py
+44 -9
@@ -816,7 +816,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
816 return image_feature
817
818 def get_multimodal_embeddings(
819 - self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
819 + self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:
820 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
821 if not modalities:
822 return None
@@ -842,8 +842,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
842 def get_input_embeddings(
843 self,
844 input_ids: torch.Tensor,
845 - multimodal_embeddings: Optional[List[Tuple[NestedTensors,
846 - str]]] = None,
845 + multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,
846 ) -> torch.Tensor:
847 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
848 if multimodal_embeddings is not None:
@@ -852,6 +851,34 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
851 [self.config.image_token_index, self.config.video_token_index])
852 return inputs_embeds
853
854 + def get_input_embeddings_v0(
855 + self,
856 + input_ids: torch.Tensor,
857 + image_input: Optional[NestedTensors] = None,
858 + video_input: Optional[NestedTensors] = None,
859 + ) -> torch.Tensor:
860 +
861 + inputs_embeds = self.get_input_embeddings(input_ids)
862 + if image_input is not None:
863 + image_embeds = self._process_image_input(image_input)
864 + inputs_embeds = merge_multimodal_embeddings(
865 + input_ids,
866 + inputs_embeds,
867 + image_embeds,
868 + placeholder_token_id=self.config.image_token_index,
869 + )
870 +
871 + if video_input is not None:
872 + video_embeds = self._process_video_pixels(video_input)
873 + inputs_embeds = merge_multimodal_embeddings(
874 + input_ids,
875 + inputs_embeds,
876 + video_embeds,
877 + placeholder_token_id=self.config.video_token_index,
878 + )
879 +
880 + return inputs_embeds
881 +
882 def forward(
883 self,
884 input_ids: torch.Tensor,
@@ -871,13 +898,21 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
898 if intermediate_tensors is not None:
899 inputs_embeds = None
900
874 - # NOTE: In v1, inputs_embeds is always generated at model runner, this
875 - # condition is for v0 compatibility.
901 + # NOTE: In v1, inputs_embeds is always generated at model runner from
902 + # `get_multimodal_embeddings` and `get_input_embeddings`, this
903 + # condition is only for v0 compatibility.
904 elif inputs_embeds is None:
877 - multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
878 - inputs_embeds = self.get_input_embeddings(input_ids,
879 - multimodal_embeddings)
880 - input_ids = None
905 + image_input = self._parse_and_validate_image_input(**kwargs)
906 + video_input = self._parse_and_validate_video_input(**kwargs)
907 +
908 + if image_input is None and video_input is None:
909 + inputs_embeds = None
910 + else:
911 + inputs_embeds = self.get_input_embeddings_v0(
912 + input_ids,
913 + image_input=image_input,
914 + video_input=video_input)
915 + input_ids = None
916
917 hidden_states = self.language_model.model(input_ids,
918 positions,
vllm/model_executor/models/qwen2_vl.py
+48 -19
@@ -55,7 +55,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
55 from vllm.multimodal import MULTIMODAL_REGISTRY
56 from vllm.multimodal.inputs import (ImageItem, ModalityData,
57 MultiModalFieldConfig, MultiModalKwargs,
58 - NestedTensors, VideoItem)
58 + VideoItem)
59 from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
60 MultiModalDataItems, MultiModalDataParser)
61 from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -1233,7 +1233,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1233 return modalities
1234
1235 def get_multimodal_embeddings(
1236 - self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
1236 + self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:
1237
1238 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1239 if not modalities:
@@ -1260,8 +1260,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1260 def get_input_embeddings(
1261 self,
1262 input_ids: torch.Tensor,
1263 - multimodal_embeddings: Optional[List[Tuple[NestedTensors,
1264 - str]]] = None,
1263 + multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,
1264 ) -> torch.Tensor:
1265 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1266 if multimodal_embeddings is not None:
@@ -1270,6 +1269,33 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1269 [self.config.image_token_id, self.config.video_token_id])
1270 return inputs_embeds
1271
1272 + def get_input_embeddings_v0(
1273 + self,
1274 + input_ids: torch.Tensor,
1275 + image_input: Optional[tuple[torch.Tensor, ...]] = None,
1276 + video_input: Optional[tuple[torch.Tensor, ...]] = None,
1277 + ) -> torch.Tensor:
1278 +
1279 + inputs_embeds = self.get_input_embeddings(input_ids)
1280 + if image_input is not None:
1281 + image_embeds = self._process_image_input(image_input)
1282 + inputs_embeds = merge_multimodal_embeddings(
1283 + input_ids,
1284 + inputs_embeds,
1285 + image_embeds,
1286 + placeholder_token_id=self.config.image_token_id,
1287 + )
1288 +
1289 + if video_input is not None:
1290 + video_embeds = self._process_video_input(video_input)
1291 + inputs_embeds = merge_multimodal_embeddings(
1292 + input_ids,
1293 + inputs_embeds,
1294 + video_embeds,
1295 + placeholder_token_id=self.config.video_token_id,
1296 + )
1297 + return inputs_embeds
1298 +
1299 def forward(
1300 self,
1301 input_ids: torch.Tensor,
@@ -1303,22 +1329,25 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1329 if intermediate_tensors is not None:
1330 inputs_embeds = None
1331
1306 - # NOTE: In v1, inputs_embeds is always generated at model runner, this
1307 - # condition is for v0 compatibility.
1332 + # NOTE: In v1, inputs_embeds is always generated at model runner from
1333 + # `get_multimodal_embeddings` and `get_input_embeddings`, this
1334 + # condition is only for v0 compatibility.
1335 elif inputs_embeds is None:
1309 - multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
1310 -
1311 - # We need to check for usage of mrope here in case there is
1312 - # multimodal data.
1313 - # TODO (ywang96): move this to model runner in V1.
1314 - if multimodal_embeddings is not None and uses_mrope(self.config):
1315 - assert positions.ndim == 2 and positions.size(0) == 3, (
1316 - "multimodal section rotary embedding requires "
1317 - f"(3, seq_len) positions, but got {positions.size()}")
1318 -
1319 - inputs_embeds = self.get_input_embeddings(input_ids,
1320 - multimodal_embeddings)
1321 - input_ids = None
1336 + image_input = self._parse_and_validate_image_input(**kwargs)
1337 + video_input = self._parse_and_validate_video_input(**kwargs)
1338 +
1339 + if image_input is None and video_input is None:
1340 + inputs_embeds = None
1341 + else:
1342 + if uses_mrope(self.config):
1343 + assert positions.ndim == 2 and positions.size(0) == 3, (
1344 + "multimodal section rotary embedding requires "
1345 + f"(3, seq_len) positions, but got {positions.size()}")
1346 + inputs_embeds = self.get_input_embeddings_v0(
1347 + input_ids,
1348 + image_input=image_input,
1349 + video_input=video_input)
1350 + input_ids = None
1351
1352 hidden_states = self.language_model.model(
1353 input_ids=input_ids,