A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Bugfix] Fix no video/image profiling edge case for `MultiModalDataParser` (#15828)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Apr 1, 2025 at 18:17 UTC
0a298ea4181a9b324afdb24876560b7e98e69bba
1 file changed
+27
-6
vllm/multimodal/parse.py
+27
-6
@@ -295,7 +295,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
295
296
297
ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
298
- ModalityDataItems[Any, Any]]
298
+ Optional[ModalityDataItems[Any, Any]]]
299
300
301
class MultiModalDataParser:
@@ -319,7 +319,15 @@ class MultiModalDataParser:
319
if isinstance(data, torch.Tensor):
320
return data.ndim == 3
321
if is_list_of(data, torch.Tensor):
322
- return len(data) == 0 or data[0].ndim == 2
322
+ return data[0].ndim == 2
323
+
324
+ return False
325
+
326
+ def _is_empty(self, data: object) -> TypeGuard[None]:
327
+ if isinstance(data, list):
328
+ return len(data) == 0
329
+ if isinstance(data, (np.ndarray, torch.Tensor)):
330
+ return data.size == 0
331
332
return False
333
@@ -341,7 +349,12 @@ class MultiModalDataParser:
349
def _parse_audio_data(
350
self,
351
data: ModalityData[AudioItem],
344
- ) -> ModalityDataItems[Any, Any]:
352
+ ) -> Optional[ModalityDataItems[Any, Any]]:
353
+ # also check single audio item with sampling rate
354
+ if self._is_empty(data) or (isinstance(data, tuple)
355
+ and self._is_empty(data[0])):
356
+ return None
357
+
358
if self._is_embeddings(data):
359
return AudioEmbeddingItems(data)
360
@@ -378,7 +391,10 @@ class MultiModalDataParser:
391
def _parse_image_data(
392
self,
393
data: ModalityData[ImageItem],
381
- ) -> ModalityDataItems[Any, Any]:
394
+ ) -> Optional[ModalityDataItems[Any, Any]]:
395
+ if self._is_empty(data):
396
+ return None
397
+
398
if self._is_embeddings(data):
399
return ImageEmbeddingItems(data)
400
@@ -396,7 +412,10 @@ class MultiModalDataParser:
412
def _parse_video_data(
413
self,
414
data: ModalityData[VideoItem],
399
- ) -> ModalityDataItems[Any, Any]:
415
+ ) -> Optional[ModalityDataItems[Any, Any]]:
416
+ if self._is_empty(data):
417
+ return None
418
+
419
if self._is_embeddings(data):
420
return VideoEmbeddingItems(data)
421
@@ -427,6 +446,8 @@ class MultiModalDataParser:
446
if k not in subparsers:
447
raise ValueError(f"Unsupported modality: {k}")
448
430
- mm_items[k] = subparsers[k](v)
449
+ # ignore empty embedding data
450
+ if (parsed_data := subparsers[k](v)) is not None:
451
+ mm_items[k] = parsed_data
452
453
return mm_items