@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8d9b6721e7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[VLM] Abstract out multi-modal data parsing in merged processor (#11620)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 30, 2024 at 23:01 UTC 8d9b6721e7f5b7d191951c6f1cd12710ffd08093
15 files changed +560 -312
.buildkite/test-pipeline.yaml
+2 -2
@@ -356,7 +356,7 @@ steps:
356 - pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
357 - pytest -v -s models/embedding/language -m 'not core_model'
358
359 -- label: Multi-Modal Models Test (Standard) # 28min
359 +- label: Multi-Modal Models Test (Standard) # 40min
360 #mirror_hardwares: [amd]
361 source_file_dependencies:
362 - vllm/
@@ -372,7 +372,7 @@ steps:
372 - pytest -v -s models/encoder_decoder/language -m core_model
373 - pytest -v -s models/encoder_decoder/vision_language -m core_model
374
375 -- label: Multi-Modal Models Test (Extended) 1 # 1h16m
375 +- label: Multi-Modal Models Test (Extended) 1 # 48m
376 optional: true
377 source_file_dependencies:
378 - vllm/
vllm/model_executor/models/chatglm.py
+2 -2
@@ -33,7 +33,7 @@ from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33 from vllm.model_executor.models.module_mapping import MultiModelKeys
34 from vllm.model_executor.sampling_metadata import SamplingMetadata
35 from vllm.multimodal import MULTIMODAL_REGISTRY
36 -from vllm.multimodal.inputs import (MultiModalData, MultiModalKwargs,
36 +from vllm.multimodal.inputs import (ModalityData, MultiModalKwargs,
37 NestedTensors)
38 from vllm.multimodal.utils import cached_get_tokenizer
39 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
@@ -54,7 +54,7 @@ def calculate_image_placeholder(vision_config):
54
55 def mm_input_mapper_for_glmv(
56 ctx: InputContext,
57 - data: MultiModalData[object],
57 + data: ModalityData[object],
58 ) -> Dict:
59 model_config = ctx.model_config
60 tokenizer = cached_get_tokenizer(
vllm/model_executor/models/llava.py
+11 -7
@@ -20,11 +20,13 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
20 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
21 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 from vllm.multimodal import MULTIMODAL_REGISTRY
23 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
24 - MultiModalFieldConfig, MultiModalInputsV2,
25 - MultiModalKwargs, NestedTensors)
23 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24 + MultiModalInputsV2, MultiModalKwargs,
25 + NestedTensors)
26 +from vllm.multimodal.parse import ImageProcessorItems
27 from vllm.multimodal.processing import (BaseMultiModalProcessor,
27 - ProcessorInputs, PromptReplacement,
28 + MultiModalDataItems, ProcessorInputs,
29 + PromptReplacement,
30 full_groupby_modality)
31 from vllm.sequence import IntermediateTensors
32
@@ -179,7 +181,9 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
181 assert isinstance(vision_config, PixtralVisionConfig)
182
183 def get_replacement_pixtral(item_idx: int):
182 - image_size = mm_items.get_image_size(item_idx)
184 + images = mm_items.get_items("image", ImageProcessorItems)
185 + image_size = images.get_image_size(item_idx)
186 +
187 (
188 num_width_tokens,
189 num_height_tokens,
@@ -591,8 +595,8 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
595
596 result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
597
594 - mm_items = self._get_mm_items(mm_data)
595 - mm_item_counts = mm_items.get_item_counts()
598 + mm_items = self._to_mm_items(mm_data)
599 + mm_item_counts = mm_items.get_all_counts()
600 mm_kwargs = result["mm_kwargs"]
601
602 # We reimplement the functionality of MLlavaProcessor from
vllm/model_executor/models/phi3v.py
+12 -7
@@ -32,12 +32,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32 from vllm.model_executor.models.clip import CLIPVisionModel
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 from vllm.multimodal import MULTIMODAL_REGISTRY
35 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
36 - MultiModalFieldConfig, MultiModalInputsV2,
37 - MultiModalKwargs, NestedTensors,
38 - PlaceholderRange)
35 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36 + MultiModalInputsV2, MultiModalKwargs,
37 + NestedTensors, PlaceholderRange)
38 +from vllm.multimodal.parse import ImageProcessorItems
39 from vllm.multimodal.processing import (BaseMultiModalProcessor,
40 - ProcessorInputs, PromptReplacement,
40 + MultiModalDataItems, ProcessorInputs,
41 + PromptReplacement,
42 _BoundPromptReplacement,
43 _PlaceholderInfo)
44 from vllm.sequence import IntermediateTensors
@@ -381,7 +382,9 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
382 assert isinstance(bos_token_id, int)
383
384 def get_replacement_phi3v(item_idx: int):
384 - image_size = mm_items.get_image_size(item_idx)
385 + images = mm_items.get_items("image", ImageProcessorItems)
386 + image_size = images.get_image_size(item_idx)
387 +
388 num_tokens = image_processor.calc_num_image_tokens_from_image_size(
389 width=image_size.width,
390 height=image_size.height,
@@ -389,12 +392,14 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
392
393 return [_IMAGE_TOKEN_ID] * num_tokens + [bos_token_id]
394
395 + num_images = mm_items.get_count("image", strict=False)
396 +
397 return [
398 PromptReplacement(
399 modality="image",
400 target=image_token,
401 replacement=get_replacement_phi3v,
397 - ) for image_token in image_tokens[:len(mm_items.images)]
402 + ) for image_token in image_tokens[:num_images]
403 ]
404
405 def _apply_prompt_replacements(
vllm/model_executor/models/qwen2_audio.py
+9 -13
@@ -20,8 +20,8 @@
20 # limitations under the License.
21 """Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22 from functools import cached_property
23 -from typing import (Any, Iterable, List, Mapping, Optional, Set, Tuple,
24 - TypedDict, Union)
23 +from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24 + Union)
25
26 import numpy as np
27 import torch
@@ -38,10 +38,12 @@ from vllm.inputs import InputContext
38 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.multimodal import MULTIMODAL_REGISTRY
41 -from vllm.multimodal.inputs import (MultiModalDataItems, MultiModalFieldConfig,
42 - MultiModalKwargs, NestedTensors)
41 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
42 + NestedTensors)
43 +from vllm.multimodal.parse import MultiModalDataParser
44 from vllm.multimodal.processing import (BaseMultiModalProcessor,
44 - ProcessorInputs, PromptReplacement)
45 + MultiModalDataItems, ProcessorInputs,
46 + PromptReplacement)
47 from vllm.sequence import IntermediateTensors
48
49 from .interfaces import SupportsMultiModal, SupportsPP
@@ -99,15 +101,9 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
101 def _get_feature_extractor(self) -> WhisperFeatureExtractor:
102 return self._get_hf_processor().feature_extractor # type: ignore
103
102 - def _get_hf_mm_data(
103 - self,
104 - mm_items: MultiModalDataItems,
105 - ) -> tuple[dict[str, Any], dict[str, Any]]:
106 - # resample audio to the model's sampling rate
104 + def _get_data_parser(self) -> MultiModalDataParser:
105 feature_extractor = self._get_feature_extractor()
108 - mm_items.resample_audios(feature_extractor.sampling_rate)
109 -
110 - return super()._get_hf_mm_data(mm_items)
106 + return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
107
108 def _call_hf_processor(
109 self,
vllm/model_executor/models/qwen2_vl.py
+72 -81
@@ -25,7 +25,6 @@ from functools import cached_property, partial
25 from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
26 Set, Tuple, Type, TypedDict, Union)
27
28 -import numpy as np
28 import torch
29 import torch.nn as nn
30 import torch.nn.functional as F
@@ -55,15 +54,16 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
54 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
55 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56 from vllm.multimodal import MULTIMODAL_REGISTRY
58 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
57 +from vllm.multimodal.inputs import (ImageItem, ModalityData,
58 MultiModalFieldConfig, MultiModalKwargs,
60 - NestedTensors)
59 + NestedTensors, VideoItem)
60 +from vllm.multimodal.parse import ModalityDataItems, MultiModalDataParser
61 from vllm.multimodal.processing import (BaseMultiModalProcessor,
62 - ProcessorInputs, PromptReplacement)
62 + MultiModalDataItems, ProcessorInputs,
63 + PromptReplacement)
64 from vllm.platforms import _Backend
65 from vllm.sequence import IntermediateTensors
66 from vllm.transformers_utils.config import uses_mrope
66 -from vllm.utils import is_list_of
67
68 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
69 from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
@@ -719,61 +719,81 @@ get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens,
719 data_type_key="video")
720
721
722 -class Qwen2VLMultiModalDataItems(MultiModalDataItems):
722 +class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
723 + dict[str, torch.Tensor]]):
724
724 - @staticmethod
725 - def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
726 - """
727 - Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
728 - """
729 - multi_data = Qwen2VLMultiModalDataItems()
730 -
731 - for k, v in data.items():
732 - # TODO: Make a separate modality for embedding inputs
733 - # to avoid confusion
734 - # yapf: disable
735 - if k == "video":
736 - # Special case since even a single item can be a list
737 - multi_data[k] = ( # type: ignore[index]
738 - v if (
739 - isinstance(v, (dict, torch.Tensor)) # type: ignore[assignment]
740 - or is_list_of(v, list)
741 - or isinstance(v[0], (np.ndarray, torch.Tensor))
742 - and v[0].ndim == 4
743 - ) else [v]
744 - )
745 - elif k in ("image", "audio"):
746 - multi_data[k] = ( # type: ignore[index]
747 - v if isinstance(v, (dict, torch.Tensor, list)) else [v]
748 - )
749 - else:
750 - multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
751 - # yapf: enable
725 + def __init__(self, data: dict, modality: str) -> None:
726 + super().__init__(data)
727
753 - return multi_data
728 + self.modality = modality
729
755 - def get_item_counts(self) -> Mapping[str, int]:
756 - return {
757 - m: (
758 - len(items[f"{m}_grid_thw"]) # type: ignore
759 - if isinstance(items, dict) else len(items))
760 - for m, items in self.items()
761 - }
730 + grid_thw = data[f"{modality}_grid_thw"]
731 + slice_idxs = [0] + grid_thw.prod(-1).cumsum_(0).tolist()
732 + self._slices = [
733 + slice(slice_idxs[i], slice_idxs[i + 1])
734 + for i in range(len(grid_thw))
735 + ]
736
763 - def has_embedding_inputs(self) -> bool:
764 - return any(
765 - isinstance(items, dict) or any(
766 - isinstance(item, torch.Tensor) for item in items)
767 - for items in self.values())
737 + def __repr__(self) -> str:
738 + return (f"{type(self).__name__}(modality={self.modality!r})")
739
740 + def get_count(self) -> int:
741 + return len(self.data[f"{self.modality}_grid_thw"])
742
770 -class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
743 + def get(self, index: int) -> dict[str, torch.Tensor]:
744 + out = {}
745 + for k, v in self.data.items():
746 + if v != f"{self.modality}_grid_thw":
747 + v = v[self._slices[index]]
748 +
749 + out[k] = v
750 +
751 + return out
752 +
753 + def get_processor_data(self) -> Mapping[str, object]:
754 + return {}
755 +
756 + def get_passthrough_data(self) -> Mapping[str, object]:
757 + return self.data
758 +
759 +
760 +class Qwen2ImageEmbeddingItems(Qwen2EmbeddingItems):
761 +
762 + def __init__(self, data: dict) -> None:
763 + super().__init__(data, "image")
764 +
765 +
766 +class Qwen2VideoEmbeddingItems(Qwen2EmbeddingItems):
767
772 - def _get_mm_items(
768 + def __init__(self, data: dict) -> None:
769 + super().__init__(data, "video")
770 +
771 +
772 +class Qwen2MultiModalDataParser(MultiModalDataParser):
773 +
774 + def _parse_image_data(
775 + self,
776 + data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
777 + ) -> ModalityDataItems[Any, Any]:
778 + if isinstance(data, dict):
779 + return Qwen2EmbeddingItems(data, modality="image")
780 +
781 + return super()._parse_image_data(data)
782 +
783 + def _parse_video_data(
784 self,
774 - mm_data: MultiModalDataDict,
775 - ) -> MultiModalDataItems:
776 - return Qwen2VLMultiModalDataItems.from_dict(mm_data)
785 + data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
786 + ) -> ModalityDataItems[Any, Any]:
787 + if isinstance(data, dict):
788 + return Qwen2EmbeddingItems(data, modality="video")
789 +
790 + return super()._parse_video_data(data)
791 +
792 +
793 +class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
794 +
795 + def _get_data_parser(self) -> MultiModalDataParser:
796 + return Qwen2MultiModalDataParser()
797
798 def _get_hf_processor(
799 self,
@@ -796,35 +816,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
816
817 return hf_processor
818
799 - def _get_hf_mm_data(
800 - self,
801 - mm_items: MultiModalDataItems,
802 - ) -> tuple[dict[str, Any], dict[str, Any]]:
803 - processor_data = dict[str, Any]()
804 - passthrough_data = dict[str, Any]()
805 -
806 - for k, v in mm_items.items():
807 - # TODO: Make a separate modality for embedding inputs
808 - # to avoid confusion
809 - if k in ("image", "video", "audio"):
810 - if isinstance(v, dict):
811 - # Pass through embedding inputs (dict)
812 - passthrough_data.update(v)
813 - elif isinstance(v, torch.Tensor) and v.ndim == 3:
814 - # Pass through embedding inputs (single)
815 - passthrough_data[f"{k}_embeds"] = [v]
816 - elif (is_list_of(v, torch.Tensor) and len(v) > 0
817 - and v[0].ndim == 2):
818 - # Pass through embedding inputs (multi)
819 - passthrough_data[f"{k}_embeds"] = v
820 - elif len(v) > 0:
821 - # Map keys to plural form, e.g.: image -> images
822 - processor_data[f"{k}s"] = v
823 - else:
824 - processor_data[k] = v
825 -
826 - return processor_data, passthrough_data
827 -
819 def _get_prompt_replacements(
820 self,
821 mm_items: MultiModalDataItems,
vllm/model_executor/models/ultravox.py
+9 -13
@@ -3,8 +3,8 @@
3
4 import math
5 from functools import cached_property, lru_cache
6 -from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
7 - Tuple, TypedDict, Union)
6 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7 + TypedDict, Union)
8
9 import numpy as np
10 import torch
@@ -24,10 +24,12 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24 from vllm.model_executor.model_loader.loader import DefaultModelLoader
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
26 from vllm.multimodal import MULTIMODAL_REGISTRY
27 -from vllm.multimodal.inputs import (MultiModalDataItems, MultiModalFieldConfig,
28 - MultiModalKwargs, NestedTensors)
27 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
28 + NestedTensors)
29 +from vllm.multimodal.parse import MultiModalDataParser
30 from vllm.multimodal.processing import (BaseMultiModalProcessor,
30 - ProcessorInputs, PromptReplacement)
31 + MultiModalDataItems, ProcessorInputs,
32 + PromptReplacement)
33 from vllm.sequence import IntermediateTensors
34 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
35 from vllm.utils import is_list_of
@@ -85,15 +87,9 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
87 hf_processor = self._get_hf_processor()
88 return hf_processor.audio_processor.feature_extractor # type: ignore
89
88 - def _get_hf_mm_data(
89 - self,
90 - mm_items: MultiModalDataItems,
91 - ) -> tuple[dict[str, Any], dict[str, Any]]:
92 - # resample audio to the model's sampling rate
90 + def _get_data_parser(self) -> MultiModalDataParser:
91 feature_extractor = self._get_feature_extractor()
94 - mm_items.resample_audios(feature_extractor.sampling_rate)
95 -
96 - return super()._get_hf_mm_data(mm_items)
92 + return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
93
94 def _call_hf_processor(
95 self,
vllm/multimodal/__init__.py
+4 -5
@@ -1,8 +1,7 @@
1 from .base import MultiModalPlaceholderMap, MultiModalPlugin
2 -from .inputs import (BatchedTensorInputs, MultiModalData,
3 - MultiModalDataBuiltins, MultiModalDataDict,
4 - MultiModalKwargs, MultiModalPlaceholderDict,
5 - NestedTensors)
2 +from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
3 + MultiModalDataDict, MultiModalKwargs,
4 + MultiModalPlaceholderDict, NestedTensors)
5 from .registry import MultiModalRegistry
6
7 MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -16,7 +15,7 @@ See also:
15
16 __all__ = [
17 "BatchedTensorInputs",
19 - "MultiModalData",
18 + "ModalityData",
19 "MultiModalDataBuiltins",
20 "MultiModalDataDict",
21 "MultiModalKwargs",
vllm/multimodal/audio.py
+2 -2
@@ -9,7 +9,7 @@ from vllm.inputs.registry import InputContext
9 from vllm.utils import PlaceholderModule
10
11 from .base import MediaIO, MultiModalPlugin
12 -from .inputs import AudioItem, MultiModalData, MultiModalKwargs
12 +from .inputs import AudioItem, ModalityData, MultiModalKwargs
13
14 try:
15 import librosa
@@ -31,7 +31,7 @@ class AudioPlugin(MultiModalPlugin):
31 def _default_input_mapper(
32 self,
33 ctx: InputContext,
34 - data: MultiModalData[AudioItem],
34 + data: ModalityData[AudioItem],
35 **mm_processor_kwargs,
36 ) -> MultiModalKwargs:
37 raise NotImplementedError("There is no default audio input mapper")
vllm/multimodal/base.py
+4 -4
@@ -15,12 +15,12 @@ if TYPE_CHECKING:
15 from vllm.config import ModelConfig
16 from vllm.sequence import SequenceGroupMetadata
17
18 -from .inputs import (MultiModalData, MultiModalDataDict, MultiModalKwargs,
18 +from .inputs import (ModalityData, MultiModalDataDict, MultiModalKwargs,
19 PlaceholderRange)
20
21 logger = init_logger(__name__)
22
23 -MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]],
23 +MultiModalInputMapper = Callable[[InputContext, ModalityData[object]],
24 MultiModalKwargs]
25 """
26 Return a dictionary to be passed as keyword arguments to
@@ -69,7 +69,7 @@ class MultiModalPlugin(ABC):
69 def _default_input_mapper(
70 self,
71 ctx: InputContext,
72 - data: MultiModalData[Any],
72 + data: ModalityData[Any],
73 **mm_processor_kwargs,
74 ) -> MultiModalKwargs:
75 """
@@ -118,7 +118,7 @@ class MultiModalPlugin(ABC):
118 def map_input(
119 self,
120 model_config: "ModelConfig",
121 - data: MultiModalData[Any],
121 + data: ModalityData[Any],
122 mm_processor_kwargs: Optional[dict[str, Any]],
123 ) -> MultiModalKwargs:
124 """
vllm/multimodal/image.py
+2 -2
@@ -13,7 +13,7 @@ from vllm.transformers_utils.processor import get_image_processor
13 from vllm.utils import is_list_of
14
15 from .base import MediaIO, MultiModalPlugin
16 -from .inputs import ImageItem, MultiModalData, MultiModalKwargs
16 +from .inputs import ImageItem, ModalityData, MultiModalKwargs
17
18 if TYPE_CHECKING:
19 from vllm.config import ModelConfig
@@ -44,7 +44,7 @@ class ImagePlugin(MultiModalPlugin):
44 def _default_input_mapper(
45 self,
46 ctx: InputContext,
47 - data: MultiModalData[ImageItem],
47 + data: ModalityData[ImageItem],
48 **mm_processor_kwargs,
49 ) -> MultiModalKwargs:
50 model_config = ctx.model_config
vllm/multimodal/inputs.py
+52 -143
@@ -2,53 +2,74 @@ from abc import ABC, abstractmethod
2 from collections import UserDict, defaultdict
3 from collections.abc import Mapping, Sequence
4 from dataclasses import dataclass
5 -from typing import (Any, Literal, NamedTuple, TypedDict, TypeVar, Union, cast,
6 - final)
5 +from typing import Any, Literal, TypedDict, TypeVar, Union, cast, final
6
7 import numpy as np
8 import torch
9 import torch.types
10 from PIL.Image import Image
11 from transformers import BatchFeature
13 -from typing_extensions import NotRequired, TypeAlias, assert_never
12 +from typing_extensions import NotRequired, TypeAlias
13
14 from vllm.utils import JSONTree, is_list_of, json_map_leaves
15
16 _T = TypeVar("_T")
17
19 -# yapf: disable
20 -ImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
18 +HfImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
19 """
20 A :class:`transformers.image_utils.ImageInput` representing a single image
21 item, which can be passed to a HuggingFace :code:`ImageProcessor`.
22 """
23
26 -VideoItem: TypeAlias = Union[
27 - list[Image],
28 - np.ndarray,
29 - torch.Tensor,
30 - list[np.ndarray],
31 - list[torch.Tensor],
32 -]
24 +HfVideoItem: TypeAlias = Union[list[Image], np.ndarray, torch.Tensor,
25 + list[np.ndarray], list[torch.Tensor]]
26 """
27 A :class:`transformers.image_utils.VideoInput` representing a single video
28 item, which can be passed to a HuggingFace :code:`VideoProcessor`.
29 """
30
38 -AudioItem: TypeAlias = Union[
39 - np.ndarray,
40 - list[float],
41 - # `(audio, sampling_rate)`: If the audio's sampling rate is different
42 - # from that expected by the model, we need to resample it.
43 - tuple[np.ndarray, float],
44 -]
31 +HfAudioItem: TypeAlias = Union[list[float], np.ndarray, torch.Tensor]
32 """
33 Represents a single audio
34 item, which can be passed to a HuggingFace :code:`AudioProcessor`.
35 """
49 -# yapf: enable
36
51 -MultiModalData: TypeAlias = Union[_T, list[_T]]
37 +ImageItem: TypeAlias = Union[HfImageItem, torch.Tensor]
38 +"""
39 +A :class:`transformers.image_utils.ImageInput` representing a single image
40 +item, which can be passed to a HuggingFace :code:`ImageProcessor`.
41 +
42 +Alternatively, a 3-D tensor or batch of 2-D tensors,
43 +which are treated as image embeddings;
44 +these are directly passed to the model without HF processing.
45 +"""
46 +
47 +VideoItem: TypeAlias = Union[HfVideoItem, torch.Tensor]
48 +"""
49 +A :class:`transformers.image_utils.VideoInput` representing a single video
50 +item, which can be passed to a HuggingFace :code:`VideoProcessor`.
51 +
52 +Alternatively, a 3-D tensor or batch of 2-D tensors,
53 +which are treated as video embeddings;
54 +these are directly passed to the model without HF processing.
55 +"""
56 +
57 +AudioItem: TypeAlias = Union[HfAudioItem, tuple[np.ndarray, float],
58 + torch.Tensor]
59 +"""
60 +Represents a single audio
61 +item, which can be passed to a HuggingFace :code:`AudioProcessor`.
62 +
63 +Alternatively, a tuple `(audio, sampling_rate)`, where the sampling rate
64 +is different from that expected by the model;
65 +these are resampled to the model's sampling rate before being processed by HF.
66 +
67 +Alternatively, a 3-D tensor or batch of 2-D tensors,
68 +which are treated as audio embeddings;
69 +these are directly passed to the model without HF processing.
70 +"""
71 +
72 +ModalityData: TypeAlias = Union[_T, list[_T]]
73 """
74 Either a single data item, or a list of data items.
75
@@ -61,17 +82,17 @@ The number of data items allowed per modality is restricted by
82 class MultiModalDataBuiltins(TypedDict, total=False):
83 """Type annotations for modality types predefined by vLLM."""
84
64 - image: MultiModalData[ImageItem]
85 + image: ModalityData[ImageItem]
86 """The input image(s)."""
87
67 - video: MultiModalData[VideoItem]
88 + video: ModalityData[VideoItem]
89 """The input video(s)."""
90
70 - audio: MultiModalData[AudioItem]
91 + audio: ModalityData[AudioItem]
92 """The input audio(s)."""
93
94
74 -MultiModalDataDict: TypeAlias = Mapping[str, MultiModalData[Any]]
95 +MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]
96 """
97 A dictionary containing an entry for each modality type to input.
98
@@ -83,123 +104,6 @@ Note:
104 """
105
106
86 -class ImageSize(NamedTuple):
87 - width: int
88 - height: int
89 -
90 -
91 -class MultiModalDataItems(UserDict[str, list[Any]]):
92 - """
93 - As :class:`MultiModalDataDict`, but normalized such that each entry
94 - corresponds to a list.
95 - """
96 -
97 - @staticmethod
98 - def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
99 - """
100 - Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
101 - """
102 - multi_data = MultiModalDataItems()
103 -
104 - for k, v in data.items():
105 - # TODO: Make a separate modality for embedding inputs
106 - # to avoid confusion
107 - # yapf: disable
108 - if k == "video":
109 - # Special case since even a single item can be a list
110 - multi_data[k] = ( # type: ignore[index]
111 - v if (
112 - isinstance(v, torch.Tensor)
113 - or is_list_of(v, list)
114 - or isinstance(v[0], (np.ndarray, torch.Tensor))
115 - and v[0].ndim == 4
116 - ) else [v]
117 - )
118 - elif k in ("image", "audio"):
119 - multi_data[k] = ( # type: ignore[index]
120 - v if isinstance(v, (torch.Tensor, list)) else [v]
121 - )
122 - else:
123 - multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
124 - # yapf: enable
125 -
126 - return multi_data
127 -
128 - # NOTE: When a field (e.g. `images`) doesn't exist, directly appending to
129 - # `self.images` doesn't update this dictionary, which may be confusing
130 - # We annotate the getter methods as `Sequence` to prevent others from
131 - # trying to update the list in this way
132 - @property
133 - def images(self) -> Sequence[ImageItem]:
134 - return self.get("image", [])
135 -
136 - @property
137 - def videos(self) -> Sequence[VideoItem]:
138 - return self.get("video", [])
139 -
140 - @property
141 - def audios(self) -> Sequence[AudioItem]:
142 - return self.get("audio", [])
143 -
144 - def get_item_counts(self) -> Mapping[str, int]:
145 - return {m: len(items) for m, items in self.items()}
146 -
147 - def has_embedding_inputs(self) -> bool:
148 - return any(
149 - any(isinstance(item, torch.Tensor) for item in items)
150 - for items in self.values())
151 -
152 - def get_image_size(self, item_idx: int) -> ImageSize:
153 - image = self.images[item_idx]
154 -
155 - if isinstance(image, Image):
156 - return ImageSize(*image.size)
157 - if isinstance(image, (np.ndarray, torch.Tensor)):
158 - _, h, w = image.shape
159 - return ImageSize(w, h)
160 -
161 - assert_never(image)
162 -
163 - def get_audio_with_sr(
164 - self,
165 - item_idx: int,
166 - *,
167 - default_sr: float,
168 - ) -> tuple[np.ndarray, float]:
169 - audio = self.audios[item_idx]
170 -
171 - if isinstance(audio, tuple):
172 - return audio
173 - if isinstance(audio, list):
174 - return np.array(audio), default_sr
175 - if isinstance(audio, np.ndarray):
176 - return audio, default_sr
177 -
178 - assert_never(audio)
179 -
180 - def resample_audios(self, new_sr: float, *, drop_sr: bool = True) -> None:
181 - """
182 - If :code:`drop_sr=True`, the audio items in this dictionary are updated
183 - to be NumPy arrays which implicitly means that their sampling rate is
184 - the same as the model's expected sampling rate; otherwise, they remain
185 - as :code:`(audio, new_sr)` tuples.
186 - """
187 - # Avoid circular import
188 - from .audio import resample_audio
189 -
190 - if not self.audios:
191 - return
192 -
193 - new_audios = []
194 - for item_idx in range(len(self.audios)):
195 - audio, sr = self.get_audio_with_sr(item_idx, default_sr=new_sr)
196 - audio = resample_audio(audio, orig_sr=sr, target_sr=new_sr)
197 -
198 - new_audios.append(audio if drop_sr else (audio, new_sr))
199 -
200 - self["audio"] = new_audios
201 -
202 -
107 class PlaceholderRange(TypedDict):
108 """
109 Placeholder location information for multi-modal data.
@@ -436,7 +340,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
340 ) -> "MultiModalKwargs":
341 data = {
342 key: items[0].field.reduce(items).data
439 - for key, items in items_by_key.items()
343 + for key, items in items_by_key.items() if len(items) > 0
344 }
345
346 return MultiModalKwargs(data,
@@ -567,6 +471,11 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
471 Get the keyword arguments corresponding to an item identified by
472 its modality and index.
473 """
474 + if modality not in self._keys_by_modality:
475 + available_modalities = set(self._keys_by_modality.keys())
476 + raise KeyError(f"Modality {modality!r} not found. "
477 + f"Available modalities: {available_modalities}")
478 +
479 keys_to_gather = self._keys_by_modality[modality]
480
481 return {
vllm/multimodal/parse.py new
+344
@@ -0,0 +1,344 @@
1 +from abc import ABC, abstractmethod
2 +from collections import UserDict
3 +from collections.abc import Callable, Iterator, Mapping, Sequence
4 +from typing import TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar
5 +
6 +import numpy as np
7 +import torch
8 +from PIL.Image import Image
9 +from typing_extensions import TypeAlias, TypeGuard, assert_never
10 +
11 +from vllm.utils import is_list_of
12 +
13 +from .audio import resample_audio
14 +from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
15 + ImageItem, ModalityData, MultiModalDataDict,
16 + NestedTensors, VideoItem)
17 +
18 +_T = TypeVar("_T")
19 +_I = TypeVar("_I")
20 +
21 +
22 +class ModalityDataItems(ABC, Generic[_T, _I]):
23 +
24 + def __init__(self, data: _T) -> None:
25 + super().__init__()
26 +
27 + self.data = data
28 +
29 + def __len__(self) -> int:
30 + return self.get_count()
31 +
32 + def __getitem__(self, index: int) -> _I:
33 + return self.get(index)
34 +
35 + if TYPE_CHECKING:
36 + # Auto-generated
37 + def __iter__(self) -> Iterator[_I]:
38 + ...
39 +
40 + @abstractmethod
41 + def get_count(self) -> int:
42 + """Get the number of data items."""
43 + raise NotImplementedError
44 +
45 + @abstractmethod
46 + def get(self, index: int) -> _I:
47 + """Get a data item by its index."""
48 + raise NotImplementedError
49 +
50 + def get_all(self) -> list[_I]:
51 + """Get all data items."""
52 + return [self.get(idx) for idx in range(self.get_count())]
53 +
54 + @abstractmethod
55 + def get_processor_data(self) -> Mapping[str, object]:
56 + """Get the data to pass to the HF processor."""
57 + raise NotImplementedError
58 +
59 + @abstractmethod
60 + def get_passthrough_data(self) -> Mapping[str, object]:
61 + """Get the data to pass directly to the model."""
62 + raise NotImplementedError
63 +
64 +
65 +class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
66 +
67 + def __init__(self, data: Sequence[_T], modality: str) -> None:
68 + super().__init__(data)
69 +
70 + self.modality = modality
71 +
72 + def __repr__(self) -> str:
73 + return (f"{type(self).__name__}(modality={self.modality!r})")
74 +
75 + def get_count(self) -> int:
76 + return len(self.data)
77 +
78 + def get(self, index: int) -> _T:
79 + return self.data[index]
80 +
81 + def get_processor_data(self) -> Mapping[str, object]:
82 + return {f"{self.modality}s": self.data}
83 +
84 + def get_passthrough_data(self) -> Mapping[str, object]:
85 + return {}
86 +
87 +
88 +class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
89 +
90 + def __init__(self, data: NestedTensors, modality: str) -> None:
91 + super().__init__(data)
92 +
93 + self.modality = modality
94 +
95 + def __repr__(self) -> str:
96 + return (f"{type(self).__name__}(modality={self.modality!r})")
97 +
98 + def get_count(self) -> int:
99 + return len(self.data)
100 +
101 + def get(self, index: int) -> object:
102 + return self.data[index]
103 +
104 + def get_processor_data(self) -> Mapping[str, object]:
105 + return {}
106 +
107 + def get_passthrough_data(self) -> Mapping[str, object]:
108 + return {f"{self.modality}_embeds": self.data}
109 +
110 +
111 +class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
112 +
113 + def __init__(self, data: Sequence[HfAudioItem]) -> None:
114 + super().__init__(data, "audio")
115 +
116 +
117 +class AudioEmbeddingItems(EmbeddingItems):
118 +
119 + def __init__(self, data: NestedTensors) -> None:
120 + super().__init__(data, "audio")
121 +
122 +
123 +class ImageSize(NamedTuple):
124 + width: int
125 + height: int
126 +
127 +
128 +class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
129 +
130 + def __init__(self, data: Sequence[HfImageItem]) -> None:
131 + super().__init__(data, "image")
132 +
133 + def get_image_size(self, item_idx: int) -> ImageSize:
134 + image = self.get(item_idx)
135 +
136 + if isinstance(image, Image):
137 + return ImageSize(*image.size)
138 + if isinstance(image, (np.ndarray, torch.Tensor)):
139 + _, h, w = image.shape
140 + return ImageSize(w, h)
141 +
142 + assert_never(image)
143 +
144 +
145 +class ImageEmbeddingItems(EmbeddingItems):
146 +
147 + def __init__(self, data: NestedTensors) -> None:
148 + super().__init__(data, "image")
149 +
150 +
151 +class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
152 +
153 + def __init__(self, data: Sequence[HfVideoItem]) -> None:
154 + super().__init__(data, "video")
155 +
156 +
157 +class VideoEmbeddingItems(EmbeddingItems):
158 +
159 + def __init__(self, data: NestedTensors) -> None:
160 + super().__init__(data, "video")
161 +
162 +
163 +_D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
164 +
165 +
166 +class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
167 + """
168 + As :class:`MultiModalDataDict`, but normalized such that each entry
169 + corresponds to a list.
170 + """
171 +
172 + def get_count(self, modality: str, *, strict: bool = True) -> int:
173 + """
174 + Get the number of data items belonging to a modality.
175 +
176 + If `strict=False`, return `0` instead of raising :exc:`KeyError`
177 + even if the modality is not found.
178 + """
179 + if modality not in self:
180 + if strict:
181 + available_modalities = set(self.keys())
182 + raise KeyError(f"Modality {modality!r} not found. "
183 + f"Available modalities: {available_modalities}")
184 +
185 + return 0
186 +
187 + return self[modality].get_count()
188 +
189 + def get_all_counts(self) -> Mapping[str, int]:
190 + """Get the number of items belonging to each modality."""
191 + return {m: items.get_count() for m, items in self.items()}
192 +
193 + def get_items(
194 + self,
195 + modality: str,
196 + typ: type[_D],
197 + ) -> _D:
198 + """
199 + Get the data items belonging to a modality,
200 + requiring that they belong to a certain type.
201 + """
202 + if modality not in self:
203 + available_modalities = set(self.keys())
204 + raise KeyError(f"Modality {modality!r} not found. "
205 + f"Available modalities: {available_modalities}")
206 +
207 + items = self[modality]
208 + if not isinstance(items, typ):
209 + raise TypeError(f"Invalid type of data items for {modality=}. "
210 + f"Expected type: {typ}, but "
211 + f"found type: {type(items)}")
212 +
213 + return items
214 +
215 +
216 +ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
217 + ModalityDataItems[Any, Any]]
218 +
219 +
220 +class MultiModalDataParser:
221 + """
222 + Parses :class:`MultiModalDataDict` into :class:`MultiModalDataItems`.
223 + """
224 +
225 + def __init__(self, *, target_sr: Optional[float] = None) -> None:
226 + super().__init__()
227 +
228 + self.target_sr = target_sr
229 +
230 + def _is_embeddings(self, data: object) -> TypeGuard[NestedTensors]:
231 + if isinstance(data, torch.Tensor):
232 + return data.ndim == 3
233 + if is_list_of(data, torch.Tensor):
234 + return len(data) == 0 or data[0].ndim == 2
235 +
236 + return False
237 +
238 + def _get_audio_with_sr(
239 + self,
240 + audio: AudioItem,
241 + ) -> tuple[np.ndarray, Optional[float]]:
242 + if isinstance(audio, tuple):
243 + return audio
244 + if isinstance(audio, list):
245 + return np.array(audio), None
246 + if isinstance(audio, np.ndarray):
247 + return audio, None
248 + if isinstance(audio, torch.Tensor):
249 + return audio.numpy(), None
250 +
251 + assert_never(audio)
252 +
253 + def _parse_audio_data(
254 + self,
255 + data: ModalityData[AudioItem],
256 + ) -> ModalityDataItems[Any, Any]:
257 + if self._is_embeddings(data):
258 + return AudioEmbeddingItems(data)
259 +
260 + if (is_list_of(data, float)
261 + or isinstance(data,
262 + (np.ndarray, torch.Tensor)) and data.ndim == 1
263 + or isinstance(data, tuple)):
264 + data_items = [data]
265 + elif isinstance(data, (np.ndarray, torch.Tensor)):
266 + data_items = [elem for elem in data]
267 + else:
268 + data_items = data
269 +
270 + new_audios = list[np.ndarray]()
271 + for data_item in data_items:
272 + audio, orig_sr = self._get_audio_with_sr(data_item)
273 + if orig_sr is None:
274 + new_audio = audio
275 + else:
276 + target_sr = self.target_sr
277 + if target_sr is None:
278 + raise RuntimeError(
279 + "Audio resampling is not supported when "
280 + "`target_sr` is not provided")
281 +
282 + new_audio = resample_audio(audio,
283 + orig_sr=orig_sr,
284 + target_sr=target_sr)
285 +
286 + new_audios.append(new_audio)
287 +
288 + return AudioProcessorItems(new_audios)
289 +
290 + def _parse_image_data(
291 + self,
292 + data: ModalityData[ImageItem],
293 + ) -> ModalityDataItems[Any, Any]:
294 + if self._is_embeddings(data):
295 + return ImageEmbeddingItems(data)
296 +
297 + if (isinstance(data, Image)
298 + or isinstance(data,
299 + (np.ndarray, torch.Tensor)) and data.ndim == 3):
300 + data_items = [data]
301 + elif isinstance(data, (np.ndarray, torch.Tensor)):
302 + data_items = [elem for elem in data]
303 + else:
304 + data_items = data
305 +
306 + return ImageProcessorItems(data_items)
307 +
308 + def _parse_video_data(
309 + self,
310 + data: ModalityData[VideoItem],
311 + ) -> ModalityDataItems[Any, Any]:
312 + if self._is_embeddings(data):
313 + return VideoEmbeddingItems(data)
314 +
315 + if (is_list_of(data, Image)
316 + or isinstance(data,
317 + (np.ndarray, torch.Tensor)) and data.ndim == 4):
318 + data_items = [data]
319 + elif isinstance(data, (np.ndarray, torch.Tensor)):
320 + data_items = [elem for elem in data]
321 + else:
322 + data_items = data
323 +
324 + return VideoProcessorItems(data_items)
325 +
326 + def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
327 + return {
328 + "audio": self._parse_audio_data,
329 + "image": self._parse_image_data,
330 + "video": self._parse_video_data,
331 + }
332 +
333 + def parse_mm_data(self,
334 + mm_data: MultiModalDataDict) -> MultiModalDataItems:
335 + subparsers = self._get_subparsers()
336 +
337 + mm_items = MultiModalDataItems()
338 + for k, v in mm_data.items():
339 + if k not in subparsers:
340 + raise ValueError(f"Unsupported modality: {k}")
341 +
342 + mm_items[k] = subparsers[k](v)
343 +
344 + return mm_items
vllm/multimodal/processing.py
+33 -29
@@ -15,11 +15,12 @@ from transformers import BatchFeature, ProcessorMixin
15 from vllm.inputs import DummyData, InputProcessingContext
16 from vllm.logger import init_logger
17 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18 -from vllm.utils import LRUCache, flatten_2d_lists, full_groupby, is_list_of
18 +from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
19
20 -from .inputs import (MultiModalDataDict, MultiModalDataItems,
21 - MultiModalFieldConfig, MultiModalFieldItem,
22 - MultiModalInputsV2, MultiModalKwargs, PlaceholderRange)
20 +from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
21 + MultiModalFieldItem, MultiModalInputsV2, MultiModalKwargs,
22 + PlaceholderRange)
23 +from .parse import MultiModalDataItems, MultiModalDataParser
24
25 logger = init_logger(__name__)
26
@@ -621,6 +622,16 @@ class BaseMultiModalProcessor(ABC):
622 ) -> MultiModalInputsV2:
623 return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
624
625 + def _get_data_parser(self) -> MultiModalDataParser:
626 + """
627 + Construct a data parser to preprocess multi-modal data items
628 + before passing them to :meth:`_get_hf_mm_data`.
629 +
630 + You can support additional modalities by creating a subclass
631 + of :class:`MultiModalDataParser` that has additional subparsers.
632 + """
633 + return MultiModalDataParser()
634 +
635 def _get_hf_processor(self) -> ProcessorMixin:
636 """
637 Subclasses can add keyword arguments to this method to accept
@@ -631,11 +642,16 @@ class BaseMultiModalProcessor(ABC):
642 def _get_tokenizer(self) -> AnyTokenizer:
643 return self.ctx.tokenizer
644
634 - def _get_mm_items(
645 + def _to_mm_items(
646 self,
647 mm_data: MultiModalDataDict,
648 ) -> MultiModalDataItems:
638 - return MultiModalDataItems.from_dict(mm_data)
649 + """
650 + Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`
651 + before passing them to :meth:`_get_hf_mm_data`.
652 + """
653 + parser = self._get_data_parser()
654 + return parser.parse_mm_data(mm_data)
655
656 @abstractmethod
657 def _get_mm_fields_config(
@@ -680,22 +696,9 @@ class BaseMultiModalProcessor(ABC):
696 processor_data = dict[str, Any]()
697 passthrough_data = dict[str, Any]()
698
683 - for k, v in mm_items.items():
684 - # TODO: Make a separate modality for embedding inputs
685 - # to avoid confusion
686 - if k in ("image", "video", "audio"):
687 - if isinstance(v, torch.Tensor) and v.ndim == 3:
688 - # Pass through embedding inputs (single)
689 - passthrough_data[f"{k}_embeds"] = [v]
690 - elif (is_list_of(v, torch.Tensor) and len(v) > 0
691 - and v[0].ndim == 2):
692 - # Pass through embedding inputs (multi)
693 - passthrough_data[f"{k}_embeds"] = v
694 - elif len(v) > 0:
695 - # Map keys to plural form, e.g.: image -> images
696 - processor_data[f"{k}s"] = v
697 - else:
698 - processor_data[k] = v
699 + for items in mm_items.values():
700 + processor_data.update(items.get_processor_data())
701 + passthrough_data.update(items.get_passthrough_data())
702
703 return processor_data, passthrough_data
704
@@ -756,7 +759,7 @@ class BaseMultiModalProcessor(ABC):
759 cached items; instead, we rely on our own prompt replacement logic
760 for the full text.
761 """
759 - mm_missing_counts = mm_missing_data_items.get_item_counts()
762 + mm_missing_counts = mm_missing_data_items.get_all_counts()
763
764 prompt_ids, _ = self._apply_hf_processor(
765 prompt_text=prompt_text,
@@ -789,7 +792,8 @@ class BaseMultiModalProcessor(ABC):
792 cache = self.cache
793 model_id = self.ctx.model_config.model
794
792 - if cache is None or mm_data_items.has_embedding_inputs():
795 + _, passthrough_data = self._get_hf_mm_data(mm_data_items)
796 + if cache is None or passthrough_data:
797 return self._apply_hf_processor(
798 prompt_text=prompt_text,
799 mm_items=mm_data_items,
@@ -812,7 +816,7 @@ class BaseMultiModalProcessor(ABC):
816 modality: [mm_data_items[modality][idx] for idx in idxs]
817 for modality, idxs in mm_missing_idxs.items()
818 }
815 - mm_missing_data_items = self._get_mm_items(mm_missing_data)
819 + mm_missing_data_items = self._to_mm_items(mm_missing_data)
820
821 prompt_ids, mm_missing_kwargs = self._apply_hf_processor_missing(
822 prompt_text=prompt_text,
@@ -852,7 +856,7 @@ class BaseMultiModalProcessor(ABC):
856 mm_merged_field_items[modality] = merged_modal_items_lst
857
858 if self.enable_sanity_checks:
855 - mm_missing_counts = mm_missing_data_items.get_item_counts()
859 + mm_missing_counts = mm_missing_data_items.get_all_counts()
860 assert all(
861 item_count == mm_missing_counts[modality]
862 for modality, item_count in mm_missing_next_idx.items()), dict(
@@ -865,7 +869,7 @@ class BaseMultiModalProcessor(ABC):
869 )
870
871 if self.enable_sanity_checks:
868 - mm_item_counts = mm_data_items.get_item_counts()
872 + mm_item_counts = mm_data_items.get_all_counts()
873
874 for modality, item_count in mm_item_counts.items():
875 for item_idx in range(item_count):
@@ -958,7 +962,7 @@ class BaseMultiModalProcessor(ABC):
962 3. Extract information about the placeholder tokens from the
963 processed token IDs.
964 """
961 - mm_items = self._get_mm_items(mm_data)
965 + mm_items = self._to_mm_items(mm_data)
966
967 prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
968 prompt_text,
@@ -975,7 +979,7 @@ class BaseMultiModalProcessor(ABC):
979
980 # If HF processor already inserts placeholder tokens,
981 # there is no need for us to insert them
978 - mm_item_counts = mm_items.get_item_counts()
982 + mm_item_counts = mm_items.get_all_counts()
983 all_placeholders = self._find_placeholders(prompt_repls, prompt_ids,
984 mm_item_counts)
985
vllm/multimodal/video.py
+2 -2
@@ -15,7 +15,7 @@ from vllm.transformers_utils.processor import get_video_processor
15 from vllm.transformers_utils.tokenizer import get_tokenizer
16 from vllm.utils import PlaceholderModule, is_list_of
17
18 -from .base import MediaIO, MultiModalData
18 +from .base import MediaIO, ModalityData
19 from .image import ImageMediaIO, ImagePlugin
20 from .inputs import MultiModalKwargs, VideoItem
21
@@ -54,7 +54,7 @@ class VideoPlugin(ImagePlugin):
54 def _default_input_mapper(
55 self,
56 ctx: InputContext,
57 - data: MultiModalData[VideoItem],
57 + data: ModalityData[VideoItem],
58 **mm_processor_kwargs,
59 ) -> MultiModalKwargs:
60 model_config = ctx.model_config