A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[VLM] Abstract out multi-modal data parsing in merged processor (#11620)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 30, 2024 at 23:01 UTC
8d9b6721e7f5b7d191951c6f1cd12710ffd08093
15 files changed
+560
-312
.buildkite/test-pipeline.yaml
+2
-2
@@ -356,7 +356,7 @@ steps:
356
- pytest -v -s models/decoder_only/language -m 'not core_model and not quant_model'
357
- pytest -v -s models/embedding/language -m 'not core_model'
358
359
-- label: Multi-Modal Models Test (Standard) # 28min
359
+- label: Multi-Modal Models Test (Standard) # 40min
360
#mirror_hardwares: [amd]
361
source_file_dependencies:
362
- vllm/
@@ -372,7 +372,7 @@ steps:
372
- pytest -v -s models/encoder_decoder/language -m core_model
373
- pytest -v -s models/encoder_decoder/vision_language -m core_model
374
375
-- label: Multi-Modal Models Test (Extended) 1 # 1h16m
375
+- label: Multi-Modal Models Test (Extended) 1 # 48m
376
optional: true
377
source_file_dependencies:
378
- vllm/
vllm/model_executor/models/chatglm.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33
from vllm.model_executor.models.module_mapping import MultiModelKeys
34
from vllm.model_executor.sampling_metadata import SamplingMetadata
35
from vllm.multimodal import MULTIMODAL_REGISTRY
36
-from vllm.multimodal.inputs import (MultiModalData, MultiModalKwargs,
36
+from vllm.multimodal.inputs import (ModalityData, MultiModalKwargs,
37
NestedTensors)
38
from vllm.multimodal.utils import cached_get_tokenizer
39
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
@@ -54,7 +54,7 @@ def calculate_image_placeholder(vision_config):
54
55
def mm_input_mapper_for_glmv(
56
ctx: InputContext,
57
- data: MultiModalData[object],
57
+ data: ModalityData[object],
58
) -> Dict:
59
model_config = ctx.model_config
60
tokenizer = cached_get_tokenizer(
vllm/model_executor/models/llava.py
+11
-7
@@ -20,11 +20,13 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
20
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.multimodal import MULTIMODAL_REGISTRY
23
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
24
- MultiModalFieldConfig, MultiModalInputsV2,
25
- MultiModalKwargs, NestedTensors)
23
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24
+ MultiModalInputsV2, MultiModalKwargs,
25
+ NestedTensors)
26
+from vllm.multimodal.parse import ImageProcessorItems
27
from vllm.multimodal.processing import (BaseMultiModalProcessor,
27
- ProcessorInputs, PromptReplacement,
28
+ MultiModalDataItems, ProcessorInputs,
29
+ PromptReplacement,
30
full_groupby_modality)
31
from vllm.sequence import IntermediateTensors
32
@@ -179,7 +181,9 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
181
assert isinstance(vision_config, PixtralVisionConfig)
182
183
def get_replacement_pixtral(item_idx: int):
182
- image_size = mm_items.get_image_size(item_idx)
184
+ images = mm_items.get_items("image", ImageProcessorItems)
185
+ image_size = images.get_image_size(item_idx)
186
+
187
(
188
num_width_tokens,
189
num_height_tokens,
@@ -591,8 +595,8 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
595
596
result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
597
594
- mm_items = self._get_mm_items(mm_data)
595
- mm_item_counts = mm_items.get_item_counts()
598
+ mm_items = self._to_mm_items(mm_data)
599
+ mm_item_counts = mm_items.get_all_counts()
600
mm_kwargs = result["mm_kwargs"]
601
602
# We reimplement the functionality of MLlavaProcessor from
vllm/model_executor/models/phi3v.py
+12
-7
@@ -32,12 +32,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32
from vllm.model_executor.models.clip import CLIPVisionModel
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
34
from vllm.multimodal import MULTIMODAL_REGISTRY
35
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
36
- MultiModalFieldConfig, MultiModalInputsV2,
37
- MultiModalKwargs, NestedTensors,
38
- PlaceholderRange)
35
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36
+ MultiModalInputsV2, MultiModalKwargs,
37
+ NestedTensors, PlaceholderRange)
38
+from vllm.multimodal.parse import ImageProcessorItems
39
from vllm.multimodal.processing import (BaseMultiModalProcessor,
40
- ProcessorInputs, PromptReplacement,
40
+ MultiModalDataItems, ProcessorInputs,
41
+ PromptReplacement,
42
_BoundPromptReplacement,
43
_PlaceholderInfo)
44
from vllm.sequence import IntermediateTensors
@@ -381,7 +382,9 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
382
assert isinstance(bos_token_id, int)
383
384
def get_replacement_phi3v(item_idx: int):
384
- image_size = mm_items.get_image_size(item_idx)
385
+ images = mm_items.get_items("image", ImageProcessorItems)
386
+ image_size = images.get_image_size(item_idx)
387
+
388
num_tokens = image_processor.calc_num_image_tokens_from_image_size(
389
width=image_size.width,
390
height=image_size.height,
@@ -389,12 +392,14 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
392
393
return [_IMAGE_TOKEN_ID] * num_tokens + [bos_token_id]
394
395
+ num_images = mm_items.get_count("image", strict=False)
396
+
397
return [
398
PromptReplacement(
399
modality="image",
400
target=image_token,
401
replacement=get_replacement_phi3v,
397
- ) for image_token in image_tokens[:len(mm_items.images)]
402
+ ) for image_token in image_tokens[:num_images]
403
]
404
405
def _apply_prompt_replacements(
vllm/model_executor/models/qwen2_audio.py
+9
-13
@@ -20,8 +20,8 @@
20
# limitations under the License.
21
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22
from functools import cached_property
23
-from typing import (Any, Iterable, List, Mapping, Optional, Set, Tuple,
24
- TypedDict, Union)
23
+from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24
+ Union)
25
26
import numpy as np
27
import torch
@@ -38,10 +38,12 @@ from vllm.inputs import InputContext
38
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.multimodal import MULTIMODAL_REGISTRY
41
-from vllm.multimodal.inputs import (MultiModalDataItems, MultiModalFieldConfig,
42
- MultiModalKwargs, NestedTensors)
41
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
42
+ NestedTensors)
43
+from vllm.multimodal.parse import MultiModalDataParser
44
from vllm.multimodal.processing import (BaseMultiModalProcessor,
44
- ProcessorInputs, PromptReplacement)
45
+ MultiModalDataItems, ProcessorInputs,
46
+ PromptReplacement)
47
from vllm.sequence import IntermediateTensors
48
49
from .interfaces import SupportsMultiModal, SupportsPP
@@ -99,15 +101,9 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
101
def _get_feature_extractor(self) -> WhisperFeatureExtractor:
102
return self._get_hf_processor().feature_extractor # type: ignore
103
102
- def _get_hf_mm_data(
103
- self,
104
- mm_items: MultiModalDataItems,
105
- ) -> tuple[dict[str, Any], dict[str, Any]]:
106
- # resample audio to the model's sampling rate
104
+ def _get_data_parser(self) -> MultiModalDataParser:
105
feature_extractor = self._get_feature_extractor()
108
- mm_items.resample_audios(feature_extractor.sampling_rate)
109
-
110
- return super()._get_hf_mm_data(mm_items)
106
+ return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
107
108
def _call_hf_processor(
109
self,
vllm/model_executor/models/qwen2_vl.py
+72
-81
@@ -25,7 +25,6 @@ from functools import cached_property, partial
25
from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
26
Set, Tuple, Type, TypedDict, Union)
27
28
-import numpy as np
28
import torch
29
import torch.nn as nn
30
import torch.nn.functional as F
@@ -55,15 +54,16 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
54
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
55
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56
from vllm.multimodal import MULTIMODAL_REGISTRY
58
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
57
+from vllm.multimodal.inputs import (ImageItem, ModalityData,
58
MultiModalFieldConfig, MultiModalKwargs,
60
- NestedTensors)
59
+ NestedTensors, VideoItem)
60
+from vllm.multimodal.parse import ModalityDataItems, MultiModalDataParser
61
from vllm.multimodal.processing import (BaseMultiModalProcessor,
62
- ProcessorInputs, PromptReplacement)
62
+ MultiModalDataItems, ProcessorInputs,
63
+ PromptReplacement)
64
from vllm.platforms import _Backend
65
from vllm.sequence import IntermediateTensors
66
from vllm.transformers_utils.config import uses_mrope
66
-from vllm.utils import is_list_of
67
68
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
69
from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
@@ -719,61 +719,81 @@ get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens,
719
data_type_key="video")
720
721
722
-class Qwen2VLMultiModalDataItems(MultiModalDataItems):
722
+class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
723
+ dict[str, torch.Tensor]]):
724
724
- @staticmethod
725
- def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
726
- """
727
- Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
728
- """
729
- multi_data = Qwen2VLMultiModalDataItems()
730
-
731
- for k, v in data.items():
732
- # TODO: Make a separate modality for embedding inputs
733
- # to avoid confusion
734
- # yapf: disable
735
- if k == "video":
736
- # Special case since even a single item can be a list
737
- multi_data[k] = ( # type: ignore[index]
738
- v if (
739
- isinstance(v, (dict, torch.Tensor)) # type: ignore[assignment]
740
- or is_list_of(v, list)
741
- or isinstance(v[0], (np.ndarray, torch.Tensor))
742
- and v[0].ndim == 4
743
- ) else [v]
744
- )
745
- elif k in ("image", "audio"):
746
- multi_data[k] = ( # type: ignore[index]
747
- v if isinstance(v, (dict, torch.Tensor, list)) else [v]
748
- )
749
- else:
750
- multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
751
- # yapf: enable
725
+ def __init__(self, data: dict, modality: str) -> None:
726
+ super().__init__(data)
727
753
- return multi_data
728
+ self.modality = modality
729
755
- def get_item_counts(self) -> Mapping[str, int]:
756
- return {
757
- m: (
758
- len(items[f"{m}_grid_thw"]) # type: ignore
759
- if isinstance(items, dict) else len(items))
760
- for m, items in self.items()
761
- }
730
+ grid_thw = data[f"{modality}_grid_thw"]
731
+ slice_idxs = [0] + grid_thw.prod(-1).cumsum_(0).tolist()
732
+ self._slices = [
733
+ slice(slice_idxs[i], slice_idxs[i + 1])
734
+ for i in range(len(grid_thw))
735
+ ]
736
763
- def has_embedding_inputs(self) -> bool:
764
- return any(
765
- isinstance(items, dict) or any(
766
- isinstance(item, torch.Tensor) for item in items)
767
- for items in self.values())
737
+ def __repr__(self) -> str:
738
+ return (f"{type(self).__name__}(modality={self.modality!r})")
739
740
+ def get_count(self) -> int:
741
+ return len(self.data[f"{self.modality}_grid_thw"])
742
770
-class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
743
+ def get(self, index: int) -> dict[str, torch.Tensor]:
744
+ out = {}
745
+ for k, v in self.data.items():
746
+ if v != f"{self.modality}_grid_thw":
747
+ v = v[self._slices[index]]
748
+
749
+ out[k] = v
750
+
751
+ return out
752
+
753
+ def get_processor_data(self) -> Mapping[str, object]:
754
+ return {}
755
+
756
+ def get_passthrough_data(self) -> Mapping[str, object]:
757
+ return self.data
758
+
759
+
760
+class Qwen2ImageEmbeddingItems(Qwen2EmbeddingItems):
761
+
762
+ def __init__(self, data: dict) -> None:
763
+ super().__init__(data, "image")
764
+
765
+
766
+class Qwen2VideoEmbeddingItems(Qwen2EmbeddingItems):
767
772
- def _get_mm_items(
768
+ def __init__(self, data: dict) -> None:
769
+ super().__init__(data, "video")
770
+
771
+
772
+class Qwen2MultiModalDataParser(MultiModalDataParser):
773
+
774
+ def _parse_image_data(
775
+ self,
776
+ data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
777
+ ) -> ModalityDataItems[Any, Any]:
778
+ if isinstance(data, dict):
779
+ return Qwen2EmbeddingItems(data, modality="image")
780
+
781
+ return super()._parse_image_data(data)
782
+
783
+ def _parse_video_data(
784
self,
774
- mm_data: MultiModalDataDict,
775
- ) -> MultiModalDataItems:
776
- return Qwen2VLMultiModalDataItems.from_dict(mm_data)
785
+ data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
786
+ ) -> ModalityDataItems[Any, Any]:
787
+ if isinstance(data, dict):
788
+ return Qwen2EmbeddingItems(data, modality="video")
789
+
790
+ return super()._parse_video_data(data)
791
+
792
+
793
+class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
794
+
795
+ def _get_data_parser(self) -> MultiModalDataParser:
796
+ return Qwen2MultiModalDataParser()
797
798
def _get_hf_processor(
799
self,
@@ -796,35 +816,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
816
817
return hf_processor
818
799
- def _get_hf_mm_data(
800
- self,
801
- mm_items: MultiModalDataItems,
802
- ) -> tuple[dict[str, Any], dict[str, Any]]:
803
- processor_data = dict[str, Any]()
804
- passthrough_data = dict[str, Any]()
805
-
806
- for k, v in mm_items.items():
807
- # TODO: Make a separate modality for embedding inputs
808
- # to avoid confusion
809
- if k in ("image", "video", "audio"):
810
- if isinstance(v, dict):
811
- # Pass through embedding inputs (dict)
812
- passthrough_data.update(v)
813
- elif isinstance(v, torch.Tensor) and v.ndim == 3:
814
- # Pass through embedding inputs (single)
815
- passthrough_data[f"{k}_embeds"] = [v]
816
- elif (is_list_of(v, torch.Tensor) and len(v) > 0
817
- and v[0].ndim == 2):
818
- # Pass through embedding inputs (multi)
819
- passthrough_data[f"{k}_embeds"] = v
820
- elif len(v) > 0:
821
- # Map keys to plural form, e.g.: image -> images
822
- processor_data[f"{k}s"] = v
823
- else:
824
- processor_data[k] = v
825
-
826
- return processor_data, passthrough_data
827
-
819
def _get_prompt_replacements(
820
self,
821
mm_items: MultiModalDataItems,
vllm/model_executor/models/ultravox.py
+9
-13
@@ -3,8 +3,8 @@
3
4
import math
5
from functools import cached_property, lru_cache
6
-from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
7
- Tuple, TypedDict, Union)
6
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7
+ TypedDict, Union)
8
9
import numpy as np
10
import torch
@@ -24,10 +24,12 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24
from vllm.model_executor.model_loader.loader import DefaultModelLoader
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
26
from vllm.multimodal import MULTIMODAL_REGISTRY
27
-from vllm.multimodal.inputs import (MultiModalDataItems, MultiModalFieldConfig,
28
- MultiModalKwargs, NestedTensors)
27
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
28
+ NestedTensors)
29
+from vllm.multimodal.parse import MultiModalDataParser
30
from vllm.multimodal.processing import (BaseMultiModalProcessor,
30
- ProcessorInputs, PromptReplacement)
31
+ MultiModalDataItems, ProcessorInputs,
32
+ PromptReplacement)
33
from vllm.sequence import IntermediateTensors
34
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
35
from vllm.utils import is_list_of
@@ -85,15 +87,9 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
87
hf_processor = self._get_hf_processor()
88
return hf_processor.audio_processor.feature_extractor # type: ignore
89
88
- def _get_hf_mm_data(
89
- self,
90
- mm_items: MultiModalDataItems,
91
- ) -> tuple[dict[str, Any], dict[str, Any]]:
92
- # resample audio to the model's sampling rate
90
+ def _get_data_parser(self) -> MultiModalDataParser:
91
feature_extractor = self._get_feature_extractor()
94
- mm_items.resample_audios(feature_extractor.sampling_rate)
95
-
96
- return super()._get_hf_mm_data(mm_items)
92
+ return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
93
94
def _call_hf_processor(
95
self,
vllm/multimodal/__init__.py
+4
-5
@@ -1,8 +1,7 @@
1
from .base import MultiModalPlaceholderMap, MultiModalPlugin
2
-from .inputs import (BatchedTensorInputs, MultiModalData,
3
- MultiModalDataBuiltins, MultiModalDataDict,
4
- MultiModalKwargs, MultiModalPlaceholderDict,
5
- NestedTensors)
2
+from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
3
+ MultiModalDataDict, MultiModalKwargs,
4
+ MultiModalPlaceholderDict, NestedTensors)
5
from .registry import MultiModalRegistry
6
7
MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -16,7 +15,7 @@ See also:
15
16
__all__ = [
17
"BatchedTensorInputs",
19
- "MultiModalData",
18
+ "ModalityData",
19
"MultiModalDataBuiltins",
20
"MultiModalDataDict",
21
"MultiModalKwargs",
vllm/multimodal/audio.py
+2
-2
@@ -9,7 +9,7 @@ from vllm.inputs.registry import InputContext
9
from vllm.utils import PlaceholderModule
10
11
from .base import MediaIO, MultiModalPlugin
12
-from .inputs import AudioItem, MultiModalData, MultiModalKwargs
12
+from .inputs import AudioItem, ModalityData, MultiModalKwargs
13
14
try:
15
import librosa
@@ -31,7 +31,7 @@ class AudioPlugin(MultiModalPlugin):
31
def _default_input_mapper(
32
self,
33
ctx: InputContext,
34
- data: MultiModalData[AudioItem],
34
+ data: ModalityData[AudioItem],
35
**mm_processor_kwargs,
36
) -> MultiModalKwargs:
37
raise NotImplementedError("There is no default audio input mapper")
vllm/multimodal/base.py
+4
-4
@@ -15,12 +15,12 @@ if TYPE_CHECKING:
15
from vllm.config import ModelConfig
16
from vllm.sequence import SequenceGroupMetadata
17
18
-from .inputs import (MultiModalData, MultiModalDataDict, MultiModalKwargs,
18
+from .inputs import (ModalityData, MultiModalDataDict, MultiModalKwargs,
19
PlaceholderRange)
20
21
logger = init_logger(__name__)
22
23
-MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]],
23
+MultiModalInputMapper = Callable[[InputContext, ModalityData[object]],
24
MultiModalKwargs]
25
"""
26
Return a dictionary to be passed as keyword arguments to
@@ -69,7 +69,7 @@ class MultiModalPlugin(ABC):
69
def _default_input_mapper(
70
self,
71
ctx: InputContext,
72
- data: MultiModalData[Any],
72
+ data: ModalityData[Any],
73
**mm_processor_kwargs,
74
) -> MultiModalKwargs:
75
"""
@@ -118,7 +118,7 @@ class MultiModalPlugin(ABC):
118
def map_input(
119
self,
120
model_config: "ModelConfig",
121
- data: MultiModalData[Any],
121
+ data: ModalityData[Any],
122
mm_processor_kwargs: Optional[dict[str, Any]],
123
) -> MultiModalKwargs:
124
"""
vllm/multimodal/image.py
+2
-2
@@ -13,7 +13,7 @@ from vllm.transformers_utils.processor import get_image_processor
13
from vllm.utils import is_list_of
14
15
from .base import MediaIO, MultiModalPlugin
16
-from .inputs import ImageItem, MultiModalData, MultiModalKwargs
16
+from .inputs import ImageItem, ModalityData, MultiModalKwargs
17
18
if TYPE_CHECKING:
19
from vllm.config import ModelConfig
@@ -44,7 +44,7 @@ class ImagePlugin(MultiModalPlugin):
44
def _default_input_mapper(
45
self,
46
ctx: InputContext,
47
- data: MultiModalData[ImageItem],
47
+ data: ModalityData[ImageItem],
48
**mm_processor_kwargs,
49
) -> MultiModalKwargs:
50
model_config = ctx.model_config
vllm/multimodal/inputs.py
+52
-143
@@ -2,53 +2,74 @@ from abc import ABC, abstractmethod
2
from collections import UserDict, defaultdict
3
from collections.abc import Mapping, Sequence
4
from dataclasses import dataclass
5
-from typing import (Any, Literal, NamedTuple, TypedDict, TypeVar, Union, cast,
6
- final)
5
+from typing import Any, Literal, TypedDict, TypeVar, Union, cast, final
6
7
import numpy as np
8
import torch
9
import torch.types
10
from PIL.Image import Image
11
from transformers import BatchFeature
13
-from typing_extensions import NotRequired, TypeAlias, assert_never
12
+from typing_extensions import NotRequired, TypeAlias
13
14
from vllm.utils import JSONTree, is_list_of, json_map_leaves
15
16
_T = TypeVar("_T")
17
19
-# yapf: disable
20
-ImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
18
+HfImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
19
"""
20
A :class:`transformers.image_utils.ImageInput` representing a single image
21
item, which can be passed to a HuggingFace :code:`ImageProcessor`.
22
"""
23
26
-VideoItem: TypeAlias = Union[
27
- list[Image],
28
- np.ndarray,
29
- torch.Tensor,
30
- list[np.ndarray],
31
- list[torch.Tensor],
32
-]
24
+HfVideoItem: TypeAlias = Union[list[Image], np.ndarray, torch.Tensor,
25
+ list[np.ndarray], list[torch.Tensor]]
26
"""
27
A :class:`transformers.image_utils.VideoInput` representing a single video
28
item, which can be passed to a HuggingFace :code:`VideoProcessor`.
29
"""
30
38
-AudioItem: TypeAlias = Union[
39
- np.ndarray,
40
- list[float],
41
- # `(audio, sampling_rate)`: If the audio's sampling rate is different
42
- # from that expected by the model, we need to resample it.
43
- tuple[np.ndarray, float],
44
-]
31
+HfAudioItem: TypeAlias = Union[list[float], np.ndarray, torch.Tensor]
32
"""
33
Represents a single audio
34
item, which can be passed to a HuggingFace :code:`AudioProcessor`.
35
"""
49
-# yapf: enable
36
51
-MultiModalData: TypeAlias = Union[_T, list[_T]]
37
+ImageItem: TypeAlias = Union[HfImageItem, torch.Tensor]
38
+"""
39
+A :class:`transformers.image_utils.ImageInput` representing a single image
40
+item, which can be passed to a HuggingFace :code:`ImageProcessor`.
41
+
42
+Alternatively, a 3-D tensor or batch of 2-D tensors,
43
+which are treated as image embeddings;
44
+these are directly passed to the model without HF processing.
45
+"""
46
+
47
+VideoItem: TypeAlias = Union[HfVideoItem, torch.Tensor]
48
+"""
49
+A :class:`transformers.image_utils.VideoInput` representing a single video
50
+item, which can be passed to a HuggingFace :code:`VideoProcessor`.
51
+
52
+Alternatively, a 3-D tensor or batch of 2-D tensors,
53
+which are treated as video embeddings;
54
+these are directly passed to the model without HF processing.
55
+"""
56
+
57
+AudioItem: TypeAlias = Union[HfAudioItem, tuple[np.ndarray, float],
58
+ torch.Tensor]
59
+"""
60
+Represents a single audio
61
+item, which can be passed to a HuggingFace :code:`AudioProcessor`.
62
+
63
+Alternatively, a tuple `(audio, sampling_rate)`, where the sampling rate
64
+is different from that expected by the model;
65
+these are resampled to the model's sampling rate before being processed by HF.
66
+
67
+Alternatively, a 3-D tensor or batch of 2-D tensors,
68
+which are treated as audio embeddings;
69
+these are directly passed to the model without HF processing.
70
+"""
71
+
72
+ModalityData: TypeAlias = Union[_T, list[_T]]
73
"""
74
Either a single data item, or a list of data items.
75
@@ -61,17 +82,17 @@ The number of data items allowed per modality is restricted by
82
class MultiModalDataBuiltins(TypedDict, total=False):
83
"""Type annotations for modality types predefined by vLLM."""
84
64
- image: MultiModalData[ImageItem]
85
+ image: ModalityData[ImageItem]
86
"""The input image(s)."""
87
67
- video: MultiModalData[VideoItem]
88
+ video: ModalityData[VideoItem]
89
"""The input video(s)."""
90
70
- audio: MultiModalData[AudioItem]
91
+ audio: ModalityData[AudioItem]
92
"""The input audio(s)."""
93
94
74
-MultiModalDataDict: TypeAlias = Mapping[str, MultiModalData[Any]]
95
+MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]
96
"""
97
A dictionary containing an entry for each modality type to input.
98
@@ -83,123 +104,6 @@ Note:
104
"""
105
106
86
-class ImageSize(NamedTuple):
87
- width: int
88
- height: int
89
-
90
-
91
-class MultiModalDataItems(UserDict[str, list[Any]]):
92
- """
93
- As :class:`MultiModalDataDict`, but normalized such that each entry
94
- corresponds to a list.
95
- """
96
-
97
- @staticmethod
98
- def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
99
- """
100
- Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
101
- """
102
- multi_data = MultiModalDataItems()
103
-
104
- for k, v in data.items():
105
- # TODO: Make a separate modality for embedding inputs
106
- # to avoid confusion
107
- # yapf: disable
108
- if k == "video":
109
- # Special case since even a single item can be a list
110
- multi_data[k] = ( # type: ignore[index]
111
- v if (
112
- isinstance(v, torch.Tensor)
113
- or is_list_of(v, list)
114
- or isinstance(v[0], (np.ndarray, torch.Tensor))
115
- and v[0].ndim == 4
116
- ) else [v]
117
- )
118
- elif k in ("image", "audio"):
119
- multi_data[k] = ( # type: ignore[index]
120
- v if isinstance(v, (torch.Tensor, list)) else [v]
121
- )
122
- else:
123
- multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
124
- # yapf: enable
125
-
126
- return multi_data
127
-
128
- # NOTE: When a field (e.g. `images`) doesn't exist, directly appending to
129
- # `self.images` doesn't update this dictionary, which may be confusing
130
- # We annotate the getter methods as `Sequence` to prevent others from
131
- # trying to update the list in this way
132
- @property
133
- def images(self) -> Sequence[ImageItem]:
134
- return self.get("image", [])
135
-
136
- @property
137
- def videos(self) -> Sequence[VideoItem]:
138
- return self.get("video", [])
139
-
140
- @property
141
- def audios(self) -> Sequence[AudioItem]:
142
- return self.get("audio", [])
143
-
144
- def get_item_counts(self) -> Mapping[str, int]:
145
- return {m: len(items) for m, items in self.items()}
146
-
147
- def has_embedding_inputs(self) -> bool:
148
- return any(
149
- any(isinstance(item, torch.Tensor) for item in items)
150
- for items in self.values())
151
-
152
- def get_image_size(self, item_idx: int) -> ImageSize:
153
- image = self.images[item_idx]
154
-
155
- if isinstance(image, Image):
156
- return ImageSize(*image.size)
157
- if isinstance(image, (np.ndarray, torch.Tensor)):
158
- _, h, w = image.shape
159
- return ImageSize(w, h)
160
-
161
- assert_never(image)
162
-
163
- def get_audio_with_sr(
164
- self,
165
- item_idx: int,
166
- *,
167
- default_sr: float,
168
- ) -> tuple[np.ndarray, float]:
169
- audio = self.audios[item_idx]
170
-
171
- if isinstance(audio, tuple):
172
- return audio
173
- if isinstance(audio, list):
174
- return np.array(audio), default_sr
175
- if isinstance(audio, np.ndarray):
176
- return audio, default_sr
177
-
178
- assert_never(audio)
179
-
180
- def resample_audios(self, new_sr: float, *, drop_sr: bool = True) -> None:
181
- """
182
- If :code:`drop_sr=True`, the audio items in this dictionary are updated
183
- to be NumPy arrays which implicitly means that their sampling rate is
184
- the same as the model's expected sampling rate; otherwise, they remain
185
- as :code:`(audio, new_sr)` tuples.
186
- """
187
- # Avoid circular import
188
- from .audio import resample_audio
189
-
190
- if not self.audios:
191
- return
192
-
193
- new_audios = []
194
- for item_idx in range(len(self.audios)):
195
- audio, sr = self.get_audio_with_sr(item_idx, default_sr=new_sr)
196
- audio = resample_audio(audio, orig_sr=sr, target_sr=new_sr)
197
-
198
- new_audios.append(audio if drop_sr else (audio, new_sr))
199
-
200
- self["audio"] = new_audios
201
-
202
-
107
class PlaceholderRange(TypedDict):
108
"""
109
Placeholder location information for multi-modal data.
@@ -436,7 +340,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
340
) -> "MultiModalKwargs":
341
data = {
342
key: items[0].field.reduce(items).data
439
- for key, items in items_by_key.items()
343
+ for key, items in items_by_key.items() if len(items) > 0
344
}
345
346
return MultiModalKwargs(data,
@@ -567,6 +471,11 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
471
Get the keyword arguments corresponding to an item identified by
472
its modality and index.
473
"""
474
+ if modality not in self._keys_by_modality:
475
+ available_modalities = set(self._keys_by_modality.keys())
476
+ raise KeyError(f"Modality {modality!r} not found. "
477
+ f"Available modalities: {available_modalities}")
478
+
479
keys_to_gather = self._keys_by_modality[modality]
480
481
return {
vllm/multimodal/parse.py
new
+344
@@ -0,0 +1,344 @@
1
+from abc import ABC, abstractmethod
2
+from collections import UserDict
3
+from collections.abc import Callable, Iterator, Mapping, Sequence
4
+from typing import TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar
5
+
6
+import numpy as np
7
+import torch
8
+from PIL.Image import Image
9
+from typing_extensions import TypeAlias, TypeGuard, assert_never
10
+
11
+from vllm.utils import is_list_of
12
+
13
+from .audio import resample_audio
14
+from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
15
+ ImageItem, ModalityData, MultiModalDataDict,
16
+ NestedTensors, VideoItem)
17
+
18
+_T = TypeVar("_T")
19
+_I = TypeVar("_I")
20
+
21
+
22
+class ModalityDataItems(ABC, Generic[_T, _I]):
23
+
24
+ def __init__(self, data: _T) -> None:
25
+ super().__init__()
26
+
27
+ self.data = data
28
+
29
+ def __len__(self) -> int:
30
+ return self.get_count()
31
+
32
+ def __getitem__(self, index: int) -> _I:
33
+ return self.get(index)
34
+
35
+ if TYPE_CHECKING:
36
+ # Auto-generated
37
+ def __iter__(self) -> Iterator[_I]:
38
+ ...
39
+
40
+ @abstractmethod
41
+ def get_count(self) -> int:
42
+ """Get the number of data items."""
43
+ raise NotImplementedError
44
+
45
+ @abstractmethod
46
+ def get(self, index: int) -> _I:
47
+ """Get a data item by its index."""
48
+ raise NotImplementedError
49
+
50
+ def get_all(self) -> list[_I]:
51
+ """Get all data items."""
52
+ return [self.get(idx) for idx in range(self.get_count())]
53
+
54
+ @abstractmethod
55
+ def get_processor_data(self) -> Mapping[str, object]:
56
+ """Get the data to pass to the HF processor."""
57
+ raise NotImplementedError
58
+
59
+ @abstractmethod
60
+ def get_passthrough_data(self) -> Mapping[str, object]:
61
+ """Get the data to pass directly to the model."""
62
+ raise NotImplementedError
63
+
64
+
65
+class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
66
+
67
+ def __init__(self, data: Sequence[_T], modality: str) -> None:
68
+ super().__init__(data)
69
+
70
+ self.modality = modality
71
+
72
+ def __repr__(self) -> str:
73
+ return (f"{type(self).__name__}(modality={self.modality!r})")
74
+
75
+ def get_count(self) -> int:
76
+ return len(self.data)
77
+
78
+ def get(self, index: int) -> _T:
79
+ return self.data[index]
80
+
81
+ def get_processor_data(self) -> Mapping[str, object]:
82
+ return {f"{self.modality}s": self.data}
83
+
84
+ def get_passthrough_data(self) -> Mapping[str, object]:
85
+ return {}
86
+
87
+
88
+class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
89
+
90
+ def __init__(self, data: NestedTensors, modality: str) -> None:
91
+ super().__init__(data)
92
+
93
+ self.modality = modality
94
+
95
+ def __repr__(self) -> str:
96
+ return (f"{type(self).__name__}(modality={self.modality!r})")
97
+
98
+ def get_count(self) -> int:
99
+ return len(self.data)
100
+
101
+ def get(self, index: int) -> object:
102
+ return self.data[index]
103
+
104
+ def get_processor_data(self) -> Mapping[str, object]:
105
+ return {}
106
+
107
+ def get_passthrough_data(self) -> Mapping[str, object]:
108
+ return {f"{self.modality}_embeds": self.data}
109
+
110
+
111
+class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
112
+
113
+ def __init__(self, data: Sequence[HfAudioItem]) -> None:
114
+ super().__init__(data, "audio")
115
+
116
+
117
+class AudioEmbeddingItems(EmbeddingItems):
118
+
119
+ def __init__(self, data: NestedTensors) -> None:
120
+ super().__init__(data, "audio")
121
+
122
+
123
+class ImageSize(NamedTuple):
124
+ width: int
125
+ height: int
126
+
127
+
128
+class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
129
+
130
+ def __init__(self, data: Sequence[HfImageItem]) -> None:
131
+ super().__init__(data, "image")
132
+
133
+ def get_image_size(self, item_idx: int) -> ImageSize:
134
+ image = self.get(item_idx)
135
+
136
+ if isinstance(image, Image):
137
+ return ImageSize(*image.size)
138
+ if isinstance(image, (np.ndarray, torch.Tensor)):
139
+ _, h, w = image.shape
140
+ return ImageSize(w, h)
141
+
142
+ assert_never(image)
143
+
144
+
145
+class ImageEmbeddingItems(EmbeddingItems):
146
+
147
+ def __init__(self, data: NestedTensors) -> None:
148
+ super().__init__(data, "image")
149
+
150
+
151
+class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
152
+
153
+ def __init__(self, data: Sequence[HfVideoItem]) -> None:
154
+ super().__init__(data, "video")
155
+
156
+
157
+class VideoEmbeddingItems(EmbeddingItems):
158
+
159
+ def __init__(self, data: NestedTensors) -> None:
160
+ super().__init__(data, "video")
161
+
162
+
163
+_D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
164
+
165
+
166
+class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
167
+ """
168
+ As :class:`MultiModalDataDict`, but normalized such that each entry
169
+ corresponds to a list.
170
+ """
171
+
172
+ def get_count(self, modality: str, *, strict: bool = True) -> int:
173
+ """
174
+ Get the number of data items belonging to a modality.
175
+
176
+ If `strict=False`, return `0` instead of raising :exc:`KeyError`
177
+ even if the modality is not found.
178
+ """
179
+ if modality not in self:
180
+ if strict:
181
+ available_modalities = set(self.keys())
182
+ raise KeyError(f"Modality {modality!r} not found. "
183
+ f"Available modalities: {available_modalities}")
184
+
185
+ return 0
186
+
187
+ return self[modality].get_count()
188
+
189
+ def get_all_counts(self) -> Mapping[str, int]:
190
+ """Get the number of items belonging to each modality."""
191
+ return {m: items.get_count() for m, items in self.items()}
192
+
193
+ def get_items(
194
+ self,
195
+ modality: str,
196
+ typ: type[_D],
197
+ ) -> _D:
198
+ """
199
+ Get the data items belonging to a modality,
200
+ requiring that they belong to a certain type.
201
+ """
202
+ if modality not in self:
203
+ available_modalities = set(self.keys())
204
+ raise KeyError(f"Modality {modality!r} not found. "
205
+ f"Available modalities: {available_modalities}")
206
+
207
+ items = self[modality]
208
+ if not isinstance(items, typ):
209
+ raise TypeError(f"Invalid type of data items for {modality=}. "
210
+ f"Expected type: {typ}, but "
211
+ f"found type: {type(items)}")
212
+
213
+ return items
214
+
215
+
216
+ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
217
+ ModalityDataItems[Any, Any]]
218
+
219
+
220
+class MultiModalDataParser:
221
+ """
222
+ Parses :class:`MultiModalDataDict` into :class:`MultiModalDataItems`.
223
+ """
224
+
225
+ def __init__(self, *, target_sr: Optional[float] = None) -> None:
226
+ super().__init__()
227
+
228
+ self.target_sr = target_sr
229
+
230
+ def _is_embeddings(self, data: object) -> TypeGuard[NestedTensors]:
231
+ if isinstance(data, torch.Tensor):
232
+ return data.ndim == 3
233
+ if is_list_of(data, torch.Tensor):
234
+ return len(data) == 0 or data[0].ndim == 2
235
+
236
+ return False
237
+
238
+ def _get_audio_with_sr(
239
+ self,
240
+ audio: AudioItem,
241
+ ) -> tuple[np.ndarray, Optional[float]]:
242
+ if isinstance(audio, tuple):
243
+ return audio
244
+ if isinstance(audio, list):
245
+ return np.array(audio), None
246
+ if isinstance(audio, np.ndarray):
247
+ return audio, None
248
+ if isinstance(audio, torch.Tensor):
249
+ return audio.numpy(), None
250
+
251
+ assert_never(audio)
252
+
253
+ def _parse_audio_data(
254
+ self,
255
+ data: ModalityData[AudioItem],
256
+ ) -> ModalityDataItems[Any, Any]:
257
+ if self._is_embeddings(data):
258
+ return AudioEmbeddingItems(data)
259
+
260
+ if (is_list_of(data, float)
261
+ or isinstance(data,
262
+ (np.ndarray, torch.Tensor)) and data.ndim == 1
263
+ or isinstance(data, tuple)):
264
+ data_items = [data]
265
+ elif isinstance(data, (np.ndarray, torch.Tensor)):
266
+ data_items = [elem for elem in data]
267
+ else:
268
+ data_items = data
269
+
270
+ new_audios = list[np.ndarray]()
271
+ for data_item in data_items:
272
+ audio, orig_sr = self._get_audio_with_sr(data_item)
273
+ if orig_sr is None:
274
+ new_audio = audio
275
+ else:
276
+ target_sr = self.target_sr
277
+ if target_sr is None:
278
+ raise RuntimeError(
279
+ "Audio resampling is not supported when "
280
+ "`target_sr` is not provided")
281
+
282
+ new_audio = resample_audio(audio,
283
+ orig_sr=orig_sr,
284
+ target_sr=target_sr)
285
+
286
+ new_audios.append(new_audio)
287
+
288
+ return AudioProcessorItems(new_audios)
289
+
290
+ def _parse_image_data(
291
+ self,
292
+ data: ModalityData[ImageItem],
293
+ ) -> ModalityDataItems[Any, Any]:
294
+ if self._is_embeddings(data):
295
+ return ImageEmbeddingItems(data)
296
+
297
+ if (isinstance(data, Image)
298
+ or isinstance(data,
299
+ (np.ndarray, torch.Tensor)) and data.ndim == 3):
300
+ data_items = [data]
301
+ elif isinstance(data, (np.ndarray, torch.Tensor)):
302
+ data_items = [elem for elem in data]
303
+ else:
304
+ data_items = data
305
+
306
+ return ImageProcessorItems(data_items)
307
+
308
+ def _parse_video_data(
309
+ self,
310
+ data: ModalityData[VideoItem],
311
+ ) -> ModalityDataItems[Any, Any]:
312
+ if self._is_embeddings(data):
313
+ return VideoEmbeddingItems(data)
314
+
315
+ if (is_list_of(data, Image)
316
+ or isinstance(data,
317
+ (np.ndarray, torch.Tensor)) and data.ndim == 4):
318
+ data_items = [data]
319
+ elif isinstance(data, (np.ndarray, torch.Tensor)):
320
+ data_items = [elem for elem in data]
321
+ else:
322
+ data_items = data
323
+
324
+ return VideoProcessorItems(data_items)
325
+
326
+ def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
327
+ return {
328
+ "audio": self._parse_audio_data,
329
+ "image": self._parse_image_data,
330
+ "video": self._parse_video_data,
331
+ }
332
+
333
+ def parse_mm_data(self,
334
+ mm_data: MultiModalDataDict) -> MultiModalDataItems:
335
+ subparsers = self._get_subparsers()
336
+
337
+ mm_items = MultiModalDataItems()
338
+ for k, v in mm_data.items():
339
+ if k not in subparsers:
340
+ raise ValueError(f"Unsupported modality: {k}")
341
+
342
+ mm_items[k] = subparsers[k](v)
343
+
344
+ return mm_items
vllm/multimodal/processing.py
+33
-29
@@ -15,11 +15,12 @@ from transformers import BatchFeature, ProcessorMixin
15
from vllm.inputs import DummyData, InputProcessingContext
16
from vllm.logger import init_logger
17
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18
-from vllm.utils import LRUCache, flatten_2d_lists, full_groupby, is_list_of
18
+from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
19
20
-from .inputs import (MultiModalDataDict, MultiModalDataItems,
21
- MultiModalFieldConfig, MultiModalFieldItem,
22
- MultiModalInputsV2, MultiModalKwargs, PlaceholderRange)
20
+from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
21
+ MultiModalFieldItem, MultiModalInputsV2, MultiModalKwargs,
22
+ PlaceholderRange)
23
+from .parse import MultiModalDataItems, MultiModalDataParser
24
25
logger = init_logger(__name__)
26
@@ -621,6 +622,16 @@ class BaseMultiModalProcessor(ABC):
622
) -> MultiModalInputsV2:
623
return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
624
625
+ def _get_data_parser(self) -> MultiModalDataParser:
626
+ """
627
+ Construct a data parser to preprocess multi-modal data items
628
+ before passing them to :meth:`_get_hf_mm_data`.
629
+
630
+ You can support additional modalities by creating a subclass
631
+ of :class:`MultiModalDataParser` that has additional subparsers.
632
+ """
633
+ return MultiModalDataParser()
634
+
635
def _get_hf_processor(self) -> ProcessorMixin:
636
"""
637
Subclasses can add keyword arguments to this method to accept
@@ -631,11 +642,16 @@ class BaseMultiModalProcessor(ABC):
642
def _get_tokenizer(self) -> AnyTokenizer:
643
return self.ctx.tokenizer
644
634
- def _get_mm_items(
645
+ def _to_mm_items(
646
self,
647
mm_data: MultiModalDataDict,
648
) -> MultiModalDataItems:
638
- return MultiModalDataItems.from_dict(mm_data)
649
+ """
650
+ Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`
651
+ before passing them to :meth:`_get_hf_mm_data`.
652
+ """
653
+ parser = self._get_data_parser()
654
+ return parser.parse_mm_data(mm_data)
655
656
@abstractmethod
657
def _get_mm_fields_config(
@@ -680,22 +696,9 @@ class BaseMultiModalProcessor(ABC):
696
processor_data = dict[str, Any]()
697
passthrough_data = dict[str, Any]()
698
683
- for k, v in mm_items.items():
684
- # TODO: Make a separate modality for embedding inputs
685
- # to avoid confusion
686
- if k in ("image", "video", "audio"):
687
- if isinstance(v, torch.Tensor) and v.ndim == 3:
688
- # Pass through embedding inputs (single)
689
- passthrough_data[f"{k}_embeds"] = [v]
690
- elif (is_list_of(v, torch.Tensor) and len(v) > 0
691
- and v[0].ndim == 2):
692
- # Pass through embedding inputs (multi)
693
- passthrough_data[f"{k}_embeds"] = v
694
- elif len(v) > 0:
695
- # Map keys to plural form, e.g.: image -> images
696
- processor_data[f"{k}s"] = v
697
- else:
698
- processor_data[k] = v
699
+ for items in mm_items.values():
700
+ processor_data.update(items.get_processor_data())
701
+ passthrough_data.update(items.get_passthrough_data())
702
703
return processor_data, passthrough_data
704
@@ -756,7 +759,7 @@ class BaseMultiModalProcessor(ABC):
759
cached items; instead, we rely on our own prompt replacement logic
760
for the full text.
761
"""
759
- mm_missing_counts = mm_missing_data_items.get_item_counts()
762
+ mm_missing_counts = mm_missing_data_items.get_all_counts()
763
764
prompt_ids, _ = self._apply_hf_processor(
765
prompt_text=prompt_text,
@@ -789,7 +792,8 @@ class BaseMultiModalProcessor(ABC):
792
cache = self.cache
793
model_id = self.ctx.model_config.model
794
792
- if cache is None or mm_data_items.has_embedding_inputs():
795
+ _, passthrough_data = self._get_hf_mm_data(mm_data_items)
796
+ if cache is None or passthrough_data:
797
return self._apply_hf_processor(
798
prompt_text=prompt_text,
799
mm_items=mm_data_items,
@@ -812,7 +816,7 @@ class BaseMultiModalProcessor(ABC):
816
modality: [mm_data_items[modality][idx] for idx in idxs]
817
for modality, idxs in mm_missing_idxs.items()
818
}
815
- mm_missing_data_items = self._get_mm_items(mm_missing_data)
819
+ mm_missing_data_items = self._to_mm_items(mm_missing_data)
820
821
prompt_ids, mm_missing_kwargs = self._apply_hf_processor_missing(
822
prompt_text=prompt_text,
@@ -852,7 +856,7 @@ class BaseMultiModalProcessor(ABC):
856
mm_merged_field_items[modality] = merged_modal_items_lst
857
858
if self.enable_sanity_checks:
855
- mm_missing_counts = mm_missing_data_items.get_item_counts()
859
+ mm_missing_counts = mm_missing_data_items.get_all_counts()
860
assert all(
861
item_count == mm_missing_counts[modality]
862
for modality, item_count in mm_missing_next_idx.items()), dict(
@@ -865,7 +869,7 @@ class BaseMultiModalProcessor(ABC):
869
)
870
871
if self.enable_sanity_checks:
868
- mm_item_counts = mm_data_items.get_item_counts()
872
+ mm_item_counts = mm_data_items.get_all_counts()
873
874
for modality, item_count in mm_item_counts.items():
875
for item_idx in range(item_count):
@@ -958,7 +962,7 @@ class BaseMultiModalProcessor(ABC):
962
3. Extract information about the placeholder tokens from the
963
processed token IDs.
964
"""
961
- mm_items = self._get_mm_items(mm_data)
965
+ mm_items = self._to_mm_items(mm_data)
966
967
prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
968
prompt_text,
@@ -975,7 +979,7 @@ class BaseMultiModalProcessor(ABC):
979
980
# If HF processor already inserts placeholder tokens,
981
# there is no need for us to insert them
978
- mm_item_counts = mm_items.get_item_counts()
982
+ mm_item_counts = mm_items.get_all_counts()
983
all_placeholders = self._find_placeholders(prompt_repls, prompt_ids,
984
mm_item_counts)
985
vllm/multimodal/video.py
+2
-2
@@ -15,7 +15,7 @@ from vllm.transformers_utils.processor import get_video_processor
15
from vllm.transformers_utils.tokenizer import get_tokenizer
16
from vllm.utils import PlaceholderModule, is_list_of
17
18
-from .base import MediaIO, MultiModalData
18
+from .base import MediaIO, ModalityData
19
from .image import ImageMediaIO, ImagePlugin
20
from .inputs import MultiModalKwargs, VideoItem
21
@@ -54,7 +54,7 @@ class VideoPlugin(ImagePlugin):
54
def _default_input_mapper(
55
self,
56
ctx: InputContext,
57
- data: MultiModalData[VideoItem],
57
+ data: ModalityData[VideoItem],
58
**mm_processor_kwargs,
59
) -> MultiModalKwargs:
60
model_config = ctx.model_config