A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix][VLM] Fix mixed-modality inference backward compatibility for V0 (#12313)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Jan 22, 2025 at 05:06 UTC
16366ee8bbdc30aad9776b74121cfc4d8f8c897d
2 files changed
+92
-28
vllm/model_executor/models/llava_onevision.py
+44
-9
@@ -816,7 +816,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
816
return image_feature
817
818
def get_multimodal_embeddings(
819
- self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
819
+ self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:
820
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
821
if not modalities:
822
return None
@@ -842,8 +842,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
842
def get_input_embeddings(
843
self,
844
input_ids: torch.Tensor,
845
- multimodal_embeddings: Optional[List[Tuple[NestedTensors,
846
- str]]] = None,
845
+ multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,
846
) -> torch.Tensor:
847
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
848
if multimodal_embeddings is not None:
@@ -852,6 +851,34 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
851
[self.config.image_token_index, self.config.video_token_index])
852
return inputs_embeds
853
854
+ def get_input_embeddings_v0(
855
+ self,
856
+ input_ids: torch.Tensor,
857
+ image_input: Optional[NestedTensors] = None,
858
+ video_input: Optional[NestedTensors] = None,
859
+ ) -> torch.Tensor:
860
+
861
+ inputs_embeds = self.get_input_embeddings(input_ids)
862
+ if image_input is not None:
863
+ image_embeds = self._process_image_input(image_input)
864
+ inputs_embeds = merge_multimodal_embeddings(
865
+ input_ids,
866
+ inputs_embeds,
867
+ image_embeds,
868
+ placeholder_token_id=self.config.image_token_index,
869
+ )
870
+
871
+ if video_input is not None:
872
+ video_embeds = self._process_video_pixels(video_input)
873
+ inputs_embeds = merge_multimodal_embeddings(
874
+ input_ids,
875
+ inputs_embeds,
876
+ video_embeds,
877
+ placeholder_token_id=self.config.video_token_index,
878
+ )
879
+
880
+ return inputs_embeds
881
+
882
def forward(
883
self,
884
input_ids: torch.Tensor,
@@ -871,13 +898,21 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
898
if intermediate_tensors is not None:
899
inputs_embeds = None
900
874
- # NOTE: In v1, inputs_embeds is always generated at model runner, this
875
- # condition is for v0 compatibility.
901
+ # NOTE: In v1, inputs_embeds is always generated at model runner from
902
+ # `get_multimodal_embeddings` and `get_input_embeddings`, this
903
+ # condition is only for v0 compatibility.
904
elif inputs_embeds is None:
877
- multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
878
- inputs_embeds = self.get_input_embeddings(input_ids,
879
- multimodal_embeddings)
880
- input_ids = None
905
+ image_input = self._parse_and_validate_image_input(**kwargs)
906
+ video_input = self._parse_and_validate_video_input(**kwargs)
907
+
908
+ if image_input is None and video_input is None:
909
+ inputs_embeds = None
910
+ else:
911
+ inputs_embeds = self.get_input_embeddings_v0(
912
+ input_ids,
913
+ image_input=image_input,
914
+ video_input=video_input)
915
+ input_ids = None
916
917
hidden_states = self.language_model.model(input_ids,
918
positions,
vllm/model_executor/models/qwen2_vl.py
+48
-19
@@ -55,7 +55,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
55
from vllm.multimodal import MULTIMODAL_REGISTRY
56
from vllm.multimodal.inputs import (ImageItem, ModalityData,
57
MultiModalFieldConfig, MultiModalKwargs,
58
- NestedTensors, VideoItem)
58
+ VideoItem)
59
from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
60
MultiModalDataItems, MultiModalDataParser)
61
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -1233,7 +1233,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1233
return modalities
1234
1235
def get_multimodal_embeddings(
1236
- self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
1236
+ self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:
1237
1238
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1239
if not modalities:
@@ -1260,8 +1260,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1260
def get_input_embeddings(
1261
self,
1262
input_ids: torch.Tensor,
1263
- multimodal_embeddings: Optional[List[Tuple[NestedTensors,
1264
- str]]] = None,
1263
+ multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,
1264
) -> torch.Tensor:
1265
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1266
if multimodal_embeddings is not None:
@@ -1270,6 +1269,33 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1269
[self.config.image_token_id, self.config.video_token_id])
1270
return inputs_embeds
1271
1272
+ def get_input_embeddings_v0(
1273
+ self,
1274
+ input_ids: torch.Tensor,
1275
+ image_input: Optional[tuple[torch.Tensor, ...]] = None,
1276
+ video_input: Optional[tuple[torch.Tensor, ...]] = None,
1277
+ ) -> torch.Tensor:
1278
+
1279
+ inputs_embeds = self.get_input_embeddings(input_ids)
1280
+ if image_input is not None:
1281
+ image_embeds = self._process_image_input(image_input)
1282
+ inputs_embeds = merge_multimodal_embeddings(
1283
+ input_ids,
1284
+ inputs_embeds,
1285
+ image_embeds,
1286
+ placeholder_token_id=self.config.image_token_id,
1287
+ )
1288
+
1289
+ if video_input is not None:
1290
+ video_embeds = self._process_video_input(video_input)
1291
+ inputs_embeds = merge_multimodal_embeddings(
1292
+ input_ids,
1293
+ inputs_embeds,
1294
+ video_embeds,
1295
+ placeholder_token_id=self.config.video_token_id,
1296
+ )
1297
+ return inputs_embeds
1298
+
1299
def forward(
1300
self,
1301
input_ids: torch.Tensor,
@@ -1303,22 +1329,25 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1329
if intermediate_tensors is not None:
1330
inputs_embeds = None
1331
1306
- # NOTE: In v1, inputs_embeds is always generated at model runner, this
1307
- # condition is for v0 compatibility.
1332
+ # NOTE: In v1, inputs_embeds is always generated at model runner from
1333
+ # `get_multimodal_embeddings` and `get_input_embeddings`, this
1334
+ # condition is only for v0 compatibility.
1335
elif inputs_embeds is None:
1309
- multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
1310
-
1311
- # We need to check for usage of mrope here in case there is
1312
- # multimodal data.
1313
- # TODO (ywang96): move this to model runner in V1.
1314
- if multimodal_embeddings is not None and uses_mrope(self.config):
1315
- assert positions.ndim == 2 and positions.size(0) == 3, (
1316
- "multimodal section rotary embedding requires "
1317
- f"(3, seq_len) positions, but got {positions.size()}")
1318
-
1319
- inputs_embeds = self.get_input_embeddings(input_ids,
1320
- multimodal_embeddings)
1321
- input_ids = None
1336
+ image_input = self._parse_and_validate_image_input(**kwargs)
1337
+ video_input = self._parse_and_validate_video_input(**kwargs)
1338
+
1339
+ if image_input is None and video_input is None:
1340
+ inputs_embeds = None
1341
+ else:
1342
+ if uses_mrope(self.config):
1343
+ assert positions.ndim == 2 and positions.size(0) == 3, (
1344
+ "multimodal section rotary embedding requires "
1345
+ f"(3, seq_len) positions, but got {positions.size()}")
1346
+ inputs_embeds = self.get_input_embeddings_v0(
1347
+ input_ids,
1348
+ image_input=image_input,
1349
+ video_input=video_input)
1350
+ input_ids = None
1351
1352
hidden_states = self.language_model.model(
1353
input_ids=input_ids,