@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e0191a95d8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[0/N] Rename `MultiModalInputs` to `MultiModalKwargs` (#10040)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 9, 2024 at 11:31 UTC e0191a95d88c454dbb989b7457a41c93cb7f7051
32 files changed +151 -121
docs/source/design/multimodal/multimodal_index.rst
+1 -1
@@ -53,7 +53,7 @@ Base Classes
53
54 .. autodata:: vllm.multimodal.MultiModalDataDict
55
56 -.. autoclass:: vllm.multimodal.MultiModalInputs
56 +.. autoclass:: vllm.multimodal.MultiModalKwargs
57 :members:
58 :show-inheritance:
59
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen.py
+2 -2
@@ -6,7 +6,7 @@ import torch
6 from PIL.Image import Image
7
8 from vllm.inputs import InputContext, token_inputs
9 -from vllm.multimodal.base import MultiModalInputs
9 +from vllm.multimodal.base import MultiModalKwargs
10 from vllm.multimodal.utils import cached_get_tokenizer
11
12 from .....conftest import IMAGE_ASSETS
@@ -96,7 +96,7 @@ def test_input_mapper_valid_mm_data(input_mapper_for_qwen,
96 mapped_img_data = input_mapper_for_qwen(qwen_vl_context, img_data)
97 # Ensure that we get the appropriately shaped pixel_values
98 # for images and image embeddings, respectively.
99 - assert isinstance(mapped_img_data, MultiModalInputs)
99 + assert isinstance(mapped_img_data, MultiModalKwargs)
100 assert "pixel_values" in mapped_img_data
101 assert mapped_img_data["pixel_values"].shape == expected_shape
102
tests/multimodal/test_base.py
+11 -11
@@ -1,6 +1,6 @@
1 import torch
2
3 -from vllm.multimodal.base import MultiModalInputs, NestedTensors
3 +from vllm.multimodal.base import MultiModalKwargs, NestedTensors
4
5
6 def assert_nested_tensors_equal(expected: NestedTensors,
@@ -13,8 +13,8 @@ def assert_nested_tensors_equal(expected: NestedTensors,
13 assert_nested_tensors_equal(expected_item, actual_item)
14
15
16 -def assert_multimodal_inputs_equal(expected: MultiModalInputs,
17 - actual: MultiModalInputs):
16 +def assert_multimodal_inputs_equal(expected: MultiModalKwargs,
17 + actual: MultiModalKwargs):
18 assert set(expected.keys()) == set(actual.keys())
19 for key in expected:
20 assert_nested_tensors_equal(expected[key], actual[key])
@@ -22,7 +22,7 @@ def assert_multimodal_inputs_equal(expected: MultiModalInputs,
22
23 def test_multimodal_input_batch_single_tensor():
24 t = torch.rand([1, 2])
25 - result = MultiModalInputs.batch([{"image": t}])
25 + result = MultiModalKwargs.batch([{"image": t}])
26 assert_multimodal_inputs_equal(result, {"image": t.unsqueeze(0)})
27
28
@@ -30,7 +30,7 @@ def test_multimodal_input_batch_multiple_tensors():
30 a = torch.rand([1, 1, 2])
31 b = torch.rand([1, 1, 2])
32 c = torch.rand([1, 1, 2])
33 - result = MultiModalInputs.batch([{"image": a}, {"image": b}, {"image": c}])
33 + result = MultiModalKwargs.batch([{"image": a}, {"image": b}, {"image": c}])
34 assert_multimodal_inputs_equal(result, {"image": torch.stack([a, b, c])})
35
36
@@ -38,7 +38,7 @@ def test_multimodal_input_batch_multiple_heterogeneous_tensors():
38 a = torch.rand([1, 2, 2])
39 b = torch.rand([1, 3, 2])
40 c = torch.rand([1, 4, 2])
41 - result = MultiModalInputs.batch([{"image": a}, {"image": b}, {"image": c}])
41 + result = MultiModalKwargs.batch([{"image": a}, {"image": b}, {"image": c}])
42 assert_multimodal_inputs_equal(result, {"image": [a, b, c]})
43
44
@@ -46,7 +46,7 @@ def test_multimodal_input_batch_nested_tensors():
46 a = torch.rand([2, 3])
47 b = torch.rand([2, 3])
48 c = torch.rand([2, 3])
49 - result = MultiModalInputs.batch([{
49 + result = MultiModalKwargs.batch([{
50 "image": [a]
51 }, {
52 "image": [b]
@@ -65,7 +65,7 @@ def test_multimodal_input_batch_heterogeneous_lists():
65 a = torch.rand([1, 2, 3])
66 b = torch.rand([1, 2, 3])
67 c = torch.rand([1, 2, 3])
68 - result = MultiModalInputs.batch([{"image": [a, b]}, {"image": [c]}])
68 + result = MultiModalKwargs.batch([{"image": [a, b]}, {"image": [c]}])
69 assert_multimodal_inputs_equal(
70 result,
71 {"image": [torch.stack([a, b]), c.unsqueeze(0)]})
@@ -76,7 +76,7 @@ def test_multimodal_input_batch_multiple_batchable_lists():
76 b = torch.rand([1, 2, 3])
77 c = torch.rand([1, 2, 3])
78 d = torch.rand([1, 2, 3])
79 - result = MultiModalInputs.batch([{"image": [a, b]}, {"image": [c, d]}])
79 + result = MultiModalKwargs.batch([{"image": [a, b]}, {"image": [c, d]}])
80 assert_multimodal_inputs_equal(
81 result,
82 {"image": torch.stack([torch.stack([a, b]),
@@ -88,8 +88,8 @@ def test_multimodal_input_batch_mixed_stacking_depths():
88 b = torch.rand([1, 3, 3])
89 c = torch.rand([1, 4, 3])
90
91 - result = MultiModalInputs.batch([{"image": [a, b]}, {"image": [c]}])
91 + result = MultiModalKwargs.batch([{"image": [a, b]}, {"image": [c]}])
92 assert_multimodal_inputs_equal(result, {"image": [[a, b], c.unsqueeze(0)]})
93
94 - result = MultiModalInputs.batch([{"image": [a]}, {"image": [b, c]}])
94 + result = MultiModalKwargs.batch([{"image": [a]}, {"image": [b, c]}])
95 assert_multimodal_inputs_equal(result, {"image": [a.unsqueeze(0), [b, c]]})
vllm/model_executor/models/chatglm.py
+2 -2
@@ -30,7 +30,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
30 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31 from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
32 from vllm.model_executor.sampling_metadata import SamplingMetadata
33 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
33 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
34 from vllm.multimodal.base import MultiModalData
35 from vllm.multimodal.utils import cached_get_tokenizer
36 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
@@ -74,7 +74,7 @@ def mm_input_mapper_for_glmv(
74 raise
75 pixel_values = raw_batch_data['images']
76
77 - return MultiModalInputs({'pixel_values': pixel_values})
77 + return MultiModalKwargs({'pixel_values': pixel_values})
78
79
80 def merge_glm_vision_embeddings(
vllm/model_executor/models/fuyu.py
+2 -2
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput
34 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
35 from vllm.model_executor.sampling_metadata import SamplingMetadata
36 from vllm.multimodal import MULTIMODAL_REGISTRY
37 -from vllm.multimodal.base import MultiModalInputs
37 +from vllm.multimodal.base import MultiModalKwargs
38 from vllm.multimodal.image import cached_get_image_processor
39 from vllm.multimodal.utils import (cached_get_tokenizer,
40 consecutive_placeholder_ranges)
@@ -218,7 +218,7 @@ def input_mapper_for_fuyu(ctx: InputContext, data: object):
218 ])
219
220 # image has been processed with prompt in input processor
221 - return MultiModalInputs({"pixel_values": data})
221 + return MultiModalKwargs({"pixel_values": data})
222
223
224 @MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_fuyu)
vllm/model_executor/models/h2ovl.py
+5 -5
@@ -16,7 +16,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
16 token_inputs)
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 -from vllm.multimodal.base import MultiModalInputs
19 +from vllm.multimodal.base import MultiModalKwargs
20 from vllm.multimodal.utils import cached_get_tokenizer
21 from vllm.utils import is_list_of
22
@@ -324,12 +324,12 @@ class H2OVLInputPipeline(InternVLInputPipeline):
324 data: object,
325 *,
326 max_dynamic_patch: Optional[int] = None,
327 - ) -> MultiModalInputs:
327 + ) -> MultiModalKwargs:
328
329 # NOTE: Preprocessing for the image data is done in the
330 # 'input_processor' function during actual inference.
331 if isinstance(data, dict):
332 - return MultiModalInputs(data)
332 + return MultiModalKwargs(data)
333
334 # The section below is only used with dummy data during
335 # memory profiling.
@@ -347,7 +347,7 @@ class H2OVLInputPipeline(InternVLInputPipeline):
347 pixel_values = [image_pixel_values_mapper(img) for img in data]
348
349 else:
350 - return MultiModalInputs({"image_embeds": data})
350 + return MultiModalKwargs({"image_embeds": data})
351 model_config = ctx.model_config
352 tokenizer = cached_get_tokenizer(
353 model_config.tokenizer,
@@ -359,7 +359,7 @@ class H2OVLInputPipeline(InternVLInputPipeline):
359 return_tensors="pt",
360 )[0]
361
362 - return MultiModalInputs({
362 + return MultiModalKwargs({
363 "pixel_values": pixel_values,
364 "image_token_id": image_token_id
365 })
vllm/model_executor/models/idefics3.py
+2 -2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
36 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
39 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
40 from vllm.multimodal.image import cached_get_image_processor
41 from vllm.sequence import IntermediateTensors, SequenceData
42 from vllm.transformers_utils.processor import cached_get_processor
@@ -127,7 +127,7 @@ def input_mapper_for_idefics3(
127 logger.error("Failed to process image (%s)", data)
128 raise
129
130 - return MultiModalInputs(batch_data)
130 + return MultiModalKwargs(batch_data)
131
132
133 def _resize_output_size(height: int,
vllm/model_executor/models/internvl.py
+3 -3
@@ -26,7 +26,7 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 InternVisionPatchModel)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.multimodal import MULTIMODAL_REGISTRY
29 -from vllm.multimodal.base import MultiModalInputs
29 +from vllm.multimodal.base import MultiModalKwargs
30 from vllm.multimodal.utils import cached_get_tokenizer
31 from vllm.sequence import IntermediateTensors
32 from vllm.utils import is_list_of
@@ -346,7 +346,7 @@ class InternVLInputPipeline:
346 # we can't stack here because images may have different num_patches
347 data = [image_pixel_values_mapper(img) for img in data]
348 else:
349 - return MultiModalInputs({"image_embeds": data})
349 + return MultiModalKwargs({"image_embeds": data})
350 model_config = ctx.model_config
351 tokenizer = cached_get_tokenizer(
352 model_config.tokenizer,
@@ -355,7 +355,7 @@ class InternVLInputPipeline:
355 add_special_tokens=False,
356 return_tensors="pt")[0]
357
358 - return MultiModalInputs({
358 + return MultiModalKwargs({
359 "pixel_values": data,
360 "image_token_id": image_token_id
361 })
vllm/model_executor/models/minicpmv.py
+2 -2
@@ -52,7 +52,7 @@ from vllm.model_executor.models.qwen2 import Qwen2Model
52 from vllm.model_executor.models.utils import LLMWrapper
53 from vllm.model_executor.sampling_metadata import SamplingMetadata
54 from vllm.multimodal import MULTIMODAL_REGISTRY
55 -from vllm.multimodal.base import MultiModalInputs
55 +from vllm.multimodal.base import MultiModalKwargs
56 from vllm.multimodal.image import cached_get_image_processor
57 from vllm.multimodal.utils import cached_get_tokenizer
58 from vllm.sequence import IntermediateTensors, SequenceData
@@ -374,7 +374,7 @@ def input_mapper_for_minicpmv(ctx: InputContext, data: object):
374 batch_data["slice_start_id"] = data[0]["slice_start_id"]
375 batch_data["slice_end_id"] = data[0]["slice_end_id"]
376
377 - return MultiModalInputs(batch_data)
377 + return MultiModalKwargs(batch_data)
378
379
380 class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
vllm/model_executor/models/mllama.py
+1 -1
@@ -1162,7 +1162,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1162
1163 def _parse_and_validate_image_input(self, **kwargs: object):
1164 # tensor with the same shape will be batched together by
1165 - # MultiModalInputs.batch, so pixel_values here can be:
1165 + # MultiModalKwargs.batch, so pixel_values here can be:
1166 # - List[List[torch.Tensor]]:
1167 # with shape (num_tiles, 3, image_res, image_res)
1168 # - List[torch.Tensor]:
vllm/model_executor/models/molmo.py
+2 -2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
40 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
41 from vllm.multimodal.utils import cached_get_tokenizer
42 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
43 SequenceData)
@@ -866,7 +866,7 @@ def image_input_mapper_for_molmo(
866 ctx: InputContext,
867 data: object,
868 ):
869 - return MultiModalInputs(data)
869 + return MultiModalKwargs(data)
870
871
872 def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
vllm/model_executor/models/pixtral.py
+5 -5
@@ -30,7 +30,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 from vllm.model_executor.models.utils import merge_multimodal_embeddings
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 -from vllm.multimodal.base import MultiModalInputs
33 +from vllm.multimodal.base import MultiModalKwargs
34 from vllm.multimodal.utils import (cached_get_tokenizer,
35 consecutive_placeholder_ranges)
36 from vllm.sequence import IntermediateTensors, SequenceData
@@ -94,8 +94,8 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
94
95
96 def input_mapper_for_pixtral(ctx: InputContext,
97 - data: object) -> MultiModalInputs:
98 - """Maps the input data to its MultiModalInputs (if any).
97 + data: object) -> MultiModalKwargs:
98 + """Maps the input data to its MultiModalKwargs (if any).
99
100 Args:
101 ctx: Context of the loaded model.
@@ -103,7 +103,7 @@ def input_mapper_for_pixtral(ctx: InputContext,
103 to pixel_values in .forward() for a visual QWenLMHeadModel model.
104
105 Returns:
106 - MultiModalInputs containing the stacked normalized images tensor or
106 + MultiModalKwargs containing the stacked normalized images tensor or
107 image embeddings.
108 """
109 # Early exit if we have provided an image to a language only Qwen model
@@ -121,7 +121,7 @@ def input_mapper_for_pixtral(ctx: InputContext,
121 dtype=torch.float16)
122 images.append(image)
123
124 - return MultiModalInputs({"images": images})
124 + return MultiModalKwargs({"images": images})
125
126
127 def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
vllm/model_executor/models/qwen.py
+6 -6
@@ -43,7 +43,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43 from vllm.model_executor.models.module_mapping import MultiModelKeys
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.multimodal import MULTIMODAL_REGISTRY
46 -from vllm.multimodal.base import MultiModalInputs
46 +from vllm.multimodal.base import MultiModalKwargs
47 from vllm.multimodal.utils import cached_get_tokenizer
48 from vllm.sequence import IntermediateTensors, SequenceData
49 from vllm.utils import is_list_of
@@ -722,8 +722,8 @@ def input_processor_for_qwen(ctx: InputContext,
722 multi_modal_data=multi_modal_data)
723
724
725 -def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
726 - """Maps the input data to its MultiModalInputs (if any).
725 +def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalKwargs:
726 + """Maps the input data to its MultiModalKwargs (if any).
727
728 Args:
729 ctx: Context of the loaded model.
@@ -731,7 +731,7 @@ def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
731 to pixel_values in .forward() for a visual QWenLMHeadModel model.
732
733 Returns:
734 - MultiModalInputs containing the stacked normalized images tensor or
734 + MultiModalKwargs containing the stacked normalized images tensor or
735 image embeddings.
736 """
737 # Early exit if we have provided an image to a language only Qwen model
@@ -740,7 +740,7 @@ def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
740 logger.warning(
741 "Images were provided but this model has no visual config; "
742 "multimodal inputs will not be forwarded to the model.")
743 - return MultiModalInputs()
743 + return MultiModalKwargs()
744
745 model_config = ctx.model_config
746 tokenizer = cached_get_tokenizer(
@@ -784,7 +784,7 @@ def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
784 data = [data]
785 transformed_images = [transform(datum) for datum in data]
786 pixel_values = torch.stack(transformed_images, dim=0)
787 - return MultiModalInputs({"pixel_values": pixel_values})
787 + return MultiModalKwargs({"pixel_values": pixel_values})
788
789
790 def build_normalization_transform(image_size: int) -> transforms.Compose:
vllm/model_executor/models/qwen2_audio.py
+4 -4
@@ -42,7 +42,7 @@ from vllm.model_executor.model_loader.weight_utils import (
42 default_weight_loader, maybe_remap_kv_scale_name)
43 from vllm.model_executor.models.qwen2 import Qwen2Model
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
45 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
46 from vllm.multimodal.utils import consecutive_placeholder_ranges
47 from vllm.sequence import IntermediateTensors, SequenceData
48
@@ -221,13 +221,13 @@ def input_processor_for_qwen2_audio(
221 def input_mapper_for_qwen2_audio(
222 ctx: InputContext,
223 multi_modal_data: Union[np.ndarray, List[np.ndarray]],
224 -) -> MultiModalInputs:
224 +) -> MultiModalKwargs:
225 """Input mapper for Qwen2-Audio."""
226 if not isinstance(multi_modal_data, list):
227 multi_modal_data = [multi_modal_data]
228
229 if len(multi_modal_data) == 0:
230 - return MultiModalInputs()
230 + return MultiModalKwargs()
231
232 processor = cached_get_processor(ctx.model_config.model)
233 audio_feature_extractor = processor.feature_extractor
@@ -254,7 +254,7 @@ def input_mapper_for_qwen2_audio(
254 logger.error("Failed to process audio (%s)", multi_modal_data)
255 raise
256
257 - return MultiModalInputs(batch_data)
257 + return MultiModalKwargs(batch_data)
258
259
260 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_audio)
vllm/model_executor/models/qwen2_vl.py
+4 -4
@@ -57,7 +57,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
57 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
58 from vllm.model_executor.models.qwen2 import Qwen2Model
59 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
60 - MultiModalInputs)
60 + MultiModalKwargs)
61 from vllm.multimodal.base import MultiModalData
62 from vllm.multimodal.image import cached_get_image_processor
63 from vllm.multimodal.utils import cached_get_tokenizer
@@ -576,10 +576,10 @@ def mm_input_mapper_for_qwen2_vl(
576 *,
577 min_pixels: Optional[int] = None,
578 max_pixels: Optional[int] = None,
579 -) -> MultiModalInputs:
579 +) -> MultiModalKwargs:
580 """Input mapper for Qwen2-VL."""
581 if data_type_key == "image" and isinstance(data, dict):
582 - return MultiModalInputs({
582 + return MultiModalKwargs({
583 "image_embeds": data.get("image_embeds"),
584 "image_grid_thw": data.get("image_grid_thw"),
585 })
@@ -613,7 +613,7 @@ def mm_input_mapper_for_qwen2_vl(
613 logger.error("Failed to process image (%s)", data)
614 raise
615
616 - return MultiModalInputs(batch_data)
616 + return MultiModalKwargs(batch_data)
617
618
619 image_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl,
vllm/model_executor/models/ultravox.py
+4 -4
@@ -24,7 +24,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
24 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25 from vllm.model_executor.model_loader.loader import DefaultModelLoader
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 -from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
27 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
28 NestedTensors)
29 from vllm.multimodal.utils import (cached_get_tokenizer,
30 consecutive_placeholder_ranges,
@@ -116,11 +116,11 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
116 data = [data]
117
118 if len(data) == 0:
119 - return MultiModalInputs()
119 + return MultiModalKwargs()
120
121 # If the audio inputs are embeddings, no need for preprocessing
122 if is_list_of(data, torch.Tensor, check="all"):
123 - return MultiModalInputs({"audio_embeds": data})
123 + return MultiModalKwargs({"audio_embeds": data})
124
125 audio_features = []
126 for audio_input in data:
@@ -154,7 +154,7 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
154 # Remove the batch dimension because we're wrapping it in a list.
155 audio_features.append(single_audio_features.squeeze(0))
156
157 - return MultiModalInputs({"audio_features": audio_features})
157 + return MultiModalKwargs({"audio_features": audio_features})
158
159
160 def input_processor_for_ultravox(ctx: InputContext, inputs: DecoderOnlyInputs):
vllm/multimodal/__init__.py
+17 -2
@@ -1,5 +1,5 @@
1 from .base import (BatchedTensorInputs, MultiModalDataBuiltins,
2 - MultiModalDataDict, MultiModalInputs,
2 + MultiModalDataDict, MultiModalKwargs,
3 MultiModalPlaceholderDict, MultiModalPlaceholderMap,
4 MultiModalPlugin, NestedTensors)
5 from .registry import MultiModalRegistry
@@ -17,7 +17,7 @@ __all__ = [
17 "BatchedTensorInputs",
18 "MultiModalDataBuiltins",
19 "MultiModalDataDict",
20 - "MultiModalInputs",
20 + "MultiModalKwargs",
21 "MultiModalPlaceholderDict",
22 "MultiModalPlaceholderMap",
23 "MultiModalPlugin",
@@ -25,3 +25,18 @@ __all__ = [
25 "MULTIMODAL_REGISTRY",
26 "MultiModalRegistry",
27 ]
28 +
29 +
30 +def __getattr__(name: str):
31 + import warnings
32 +
33 + if name == "MultiModalInputs":
34 + msg = ("MultiModalInputs has been renamed to MultiModalKwargs. "
35 + "The original name will take another meaning in an upcoming "
36 + "version.")
37 +
38 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
39 +
40 + return MultiModalKwargs
41 +
42 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/multimodal/audio.py
+2 -2
@@ -1,5 +1,5 @@
1 from vllm.inputs.registry import InputContext
2 -from vllm.multimodal.base import MultiModalInputs, MultiModalPlugin
2 +from vllm.multimodal.base import MultiModalKwargs, MultiModalPlugin
3
4
5 class AudioPlugin(MultiModalPlugin):
@@ -9,7 +9,7 @@ class AudioPlugin(MultiModalPlugin):
9 return "audio"
10
11 def _default_input_mapper(self, ctx: InputContext, data: object,
12 - **mm_processor_kwargs) -> MultiModalInputs:
12 + **mm_processor_kwargs) -> MultiModalKwargs:
13 raise NotImplementedError("There is no default audio input mapper")
14
15 def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
vllm/multimodal/base.py
+24 -9
@@ -30,15 +30,15 @@ Uses a list instead of a tensor if the dimensions of each element do not match.
30 BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors]
31 """
32 A dictionary containing nested tensors which have been batched via
33 -:meth:`MultiModalInputs.batch`.
33 +:meth:`MultiModalKwargs.batch`.
34 """
35
36
37 -class _MultiModalInputsBase(UserDict[str, NestedTensors]):
37 +class _MultiModalKwargsBase(UserDict[str, NestedTensors]):
38 pass
39
40
41 -class MultiModalInputs(_MultiModalInputsBase):
41 +class MultiModalKwargs(_MultiModalKwargsBase):
42 """
43 A dictionary that represents the keyword arguments to
44 :meth:`~torch.nn.Module.forward`.
@@ -58,7 +58,7 @@ class MultiModalInputs(_MultiModalInputsBase):
58 if isinstance(nested_tensors, (int, float)):
59 return torch.tensor(nested_tensors)
60
61 - stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors]
61 + stacked = [MultiModalKwargs._try_stack(t) for t in nested_tensors]
62 if not is_list_of(stacked, torch.Tensor, check="all"):
63 # Only tensors (not lists) can be stacked.
64 return stacked
@@ -71,7 +71,7 @@ class MultiModalInputs(_MultiModalInputsBase):
71 return torch.stack(tensors_)
72
73 @staticmethod
74 - def batch(inputs_list: List["MultiModalInputs"]) -> BatchedTensorInputs:
74 + def batch(inputs_list: List["MultiModalKwargs"]) -> BatchedTensorInputs:
75 """
76 Batch multiple inputs together into a dictionary.
77
@@ -95,7 +95,7 @@ class MultiModalInputs(_MultiModalInputsBase):
95 item_lists[k].append(v)
96
97 return {
98 - k: MultiModalInputs._try_stack(item_list)
98 + k: MultiModalKwargs._try_stack(item_list)
99 for k, item_list in item_lists.items()
100 }
101
@@ -177,7 +177,7 @@ A dictionary containing placeholder ranges.
177 """
178
179 MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]],
180 - MultiModalInputs]
180 + MultiModalKwargs]
181 """
182 Return a dictionary to be passed as keyword arguments to
183 :meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
@@ -226,7 +226,7 @@ class MultiModalPlugin(ABC):
226 ctx: InputContext,
227 data: MultiModalData[object],
228 **mm_processor_kwargs,
229 - ) -> MultiModalInputs:
229 + ) -> MultiModalKwargs:
230 """
231 Return a dictionary to be passed as keyword arguments to
232 :meth:`~torch.nn.Module.forward`. This is similar in concept to
@@ -275,7 +275,7 @@ class MultiModalPlugin(ABC):
275 model_config: "ModelConfig",
276 data: MultiModalData[object],
277 mm_processor_kwargs: Dict[str, Any],
278 - ) -> MultiModalInputs:
278 + ) -> MultiModalKwargs:
279 """
280 Transform the data into a dictionary of model inputs using the
281 input mapper registered for that model.
@@ -585,3 +585,18 @@ class MultiModalPlaceholderMap:
585
586 return MultiModalPlaceholderMap.IndexMap(src=src_indices,
587 dest=dest_indices)
588 +
589 +
590 +def __getattr__(name: str):
591 + import warnings
592 +
593 + if name == "MultiModalInputs":
594 + msg = ("MultiModalInputs has been renamed to MultiModalKwargs. "
595 + "The original name will take another meaning in an upcoming "
596 + "version.")
597 +
598 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
599 +
600 + return MultiModalKwargs
601 +
602 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
vllm/multimodal/image.py
+5 -5
@@ -10,7 +10,7 @@ from vllm.logger import init_logger
10 from vllm.transformers_utils.processor import get_image_processor
11 from vllm.utils import is_list_of
12
13 -from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
13 +from .base import MultiModalData, MultiModalKwargs, MultiModalPlugin
14
15 if TYPE_CHECKING:
16 from vllm.config import ModelConfig
@@ -43,12 +43,12 @@ class ImagePlugin(MultiModalPlugin):
43 ctx: InputContext,
44 data: MultiModalData[object],
45 **mm_processor_kwargs,
46 - ) -> MultiModalInputs:
46 + ) -> MultiModalKwargs:
47 model_config = ctx.model_config
48
49 # Processed by input processor
50 if isinstance(data, BatchFeature):
51 - return MultiModalInputs(data.data)
51 + return MultiModalKwargs(data.data)
52
53 # PIL image
54 if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
@@ -78,11 +78,11 @@ class ImagePlugin(MultiModalPlugin):
78 type(image_processor).__name__)
79 raise
80
81 - return MultiModalInputs(batch_data)
81 + return MultiModalKwargs(batch_data)
82
83 # Image embedding
84 elif isinstance(data, torch.Tensor) or is_list_of(data, torch.Tensor):
85 - return MultiModalInputs({"image_embeds": data})
85 + return MultiModalKwargs({"image_embeds": data})
86
87 raise TypeError(f"Invalid image type: {type(data)}")
88
vllm/multimodal/registry.py
+3 -3
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Sequence
5 from vllm.logger import init_logger
6
7 from .audio import AudioPlugin
8 -from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
8 +from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalKwargs,
9 MultiModalPlugin, MultiModalTokensCalc, NestedTensors)
10 from .image import ImagePlugin
11 from .video import VideoPlugin
@@ -103,7 +103,7 @@ class MultiModalRegistry:
103 model_config: "ModelConfig",
104 data: MultiModalDataDict,
105 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
106 - ) -> MultiModalInputs:
106 + ) -> MultiModalKwargs:
107 """
108 Apply an input mapper to the data passed to the model.
109
@@ -139,7 +139,7 @@ class MultiModalRegistry:
139
140 merged_dict[input_key] = input_tensor
141
142 - return MultiModalInputs(merged_dict)
142 + return MultiModalKwargs(merged_dict)
143
144 def create_input_mapper(self, model_config: "ModelConfig"):
145 """
vllm/multimodal/video.py
+3 -3
@@ -9,7 +9,7 @@ from vllm.transformers_utils.processor import get_video_processor
9 from vllm.transformers_utils.tokenizer import get_tokenizer
10 from vllm.utils import is_list_of
11
12 -from .base import MultiModalData, MultiModalInputs
12 +from .base import MultiModalData, MultiModalKwargs
13 from .image import ImagePlugin
14
15 if TYPE_CHECKING:
@@ -55,7 +55,7 @@ class VideoPlugin(ImagePlugin):
55 ctx: InputContext,
56 data: MultiModalData[object],
57 **mm_processor_kwargs,
58 - ) -> MultiModalInputs:
58 + ) -> MultiModalKwargs:
59 model_config = ctx.model_config
60
61 if isinstance(data, list) and len(data) == 1:
@@ -79,7 +79,7 @@ class VideoPlugin(ImagePlugin):
79 logger.error("Failed to process video (%s)", data)
80 raise
81
82 - return MultiModalInputs(batch_data)
82 + return MultiModalKwargs(batch_data)
83
84 raise TypeError(f"Invalid video type: {type(data)}")
85
vllm/spec_decode/draft_model_runner.py
+2 -2
@@ -18,7 +18,7 @@ except (ModuleNotFoundError, ImportError) as err:
18 "CUDA and ROCm flash attention backend.") from err
19
20 from vllm.logger import init_logger
21 -from vllm.multimodal import MultiModalInputs
21 +from vllm.multimodal import MultiModalKwargs
22 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
23 from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
24 ModelRunner)
@@ -280,7 +280,7 @@ class TP1DraftModelRunner(ModelRunner):
280 kv_caches=kv_caches,
281 attn_metadata=model_input.attn_metadata,
282 intermediate_tensors=intermediate_tensors,
283 - **MultiModalInputs.as_kwargs(multi_modal_kwargs,
283 + **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
284 device=self.device),
285 **kwargs,
286 )
vllm/worker/cpu_enc_dec_model_runner.py
+2 -2
@@ -5,7 +5,7 @@ import torch
5
6 from vllm.attention import AttentionMetadata
7 from vllm.model_executor.layers.sampler import SamplerOutput
8 -from vllm.multimodal import MultiModalInputs
8 +from vllm.multimodal import MultiModalKwargs
9 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
10 from vllm.utils import make_tensor_with_pad
11 from vllm.worker.cpu_model_runner import (CPUModelRunner,
@@ -287,7 +287,7 @@ class CPUEncoderDecoderModelRunner(CPUModelRunner):
287 kv_caches,
288 "attn_metadata":
289 model_input.attn_metadata,
290 - **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
290 + **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
291 device=self.device),
292 "intermediate_tensors":
293 intermediate_tensors,
vllm/worker/cpu_model_runner.py
+5 -5
@@ -15,7 +15,7 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
15 from vllm.model_executor.layers.sampler import SamplerOutput
16 from vllm.model_executor.model_loader import get_model
17 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
18 - MultiModalInputs, MultiModalPlaceholderMap)
18 + MultiModalKwargs, MultiModalPlaceholderMap)
19 from vllm.sequence import (IntermediateTensors, SequenceData,
20 SequenceGroupMetadata)
21 from vllm.utils import make_tensor_with_pad
@@ -200,7 +200,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
200
201 slot_mapping: List[int] = []
202 seq_lens: List[int] = []
203 - multi_modal_inputs_list: List[MultiModalInputs] = []
203 + multi_model_kwargs_list: List[MultiModalKwargs] = []
204 multi_modal_placeholder_maps: Dict[
205 str,
206 MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
@@ -225,7 +225,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
225 ._compute_multi_modal_input(
226 seq_group_metadata, seq_data, computed_len,
227 seq_group_metadata.mm_processor_kwargs)
228 - multi_modal_inputs_list.append(mm_kwargs)
228 + multi_model_kwargs_list.append(mm_kwargs)
229 for modality, placeholder_map in placeholder_maps.items():
230 multi_modal_placeholder_maps[modality].extend(
231 placeholder_map)
@@ -297,7 +297,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
297 multi_modal_placeholder_index_maps=placeholder_index_maps,
298 )
299
300 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
300 + multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
301
302 return (input_tokens, input_positions, attn_metadata, seq_lens,
303 multi_modal_kwargs)
@@ -520,7 +520,7 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
520 kv_caches,
521 "attn_metadata":
522 model_input.attn_metadata,
523 - **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
523 + **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
524 device=self.device),
525 "intermediate_tensors":
526 intermediate_tensors,
vllm/worker/embedding_model_runner.py
+2 -2
@@ -8,7 +8,7 @@ from vllm.distributed import get_pp_group
8 from vllm.forward_context import set_forward_context
9 from vllm.logger import init_logger
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 -from vllm.multimodal import MultiModalInputs
11 +from vllm.multimodal import MultiModalKwargs
12 from vllm.pooling_params import PoolingParams
13 from vllm.sequence import (IntermediateTensors, PoolerOutput, SequenceData,
14 SequenceGroupMetadata)
@@ -104,7 +104,7 @@ class EmbeddingModelRunner(
104 kv_caches=kv_caches,
105 attn_metadata=model_input.attn_metadata,
106 intermediate_tensors=intermediate_tensors,
107 - **MultiModalInputs.as_kwargs(multi_modal_kwargs,
107 + **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
108 device=self.device))
109
110 if (self.observability_config is not None
vllm/worker/enc_dec_model_runner.py
+2 -2
@@ -18,7 +18,7 @@ from vllm.logger import init_logger
18 from vllm.model_executor import SamplingMetadata
19 from vllm.model_executor.layers.sampler import SamplerOutput
20 from vllm.model_executor.model_loader.utils import get_architecture_class_name
21 -from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
21 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
22 MultiModalRegistry)
23 from vllm.sampling_params import SamplingParams
24 from vllm.sequence import (IntermediateTensors, PoolerOutput,
@@ -206,7 +206,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
206 kv_caches=kv_caches,
207 attn_metadata=model_input.attn_metadata,
208 intermediate_tensors=intermediate_tensors,
209 - **MultiModalInputs.as_kwargs(multi_modal_kwargs,
209 + **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
210 device=self.device),
211 **seqlen_agnostic_kwargs)
212
vllm/worker/hpu_model_runner.py
+4 -4
@@ -36,7 +36,7 @@ from vllm.model_executor import SamplingMetadata
36 from vllm.model_executor.layers.sampler import SamplerOutput
37 from vllm.model_executor.model_loader import get_model
38 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
39 - MultiModalInputs)
39 + MultiModalKwargs)
40 from vllm.sampling_params import SamplingParams
41 from vllm.sequence import (IntermediateTensors, SequenceData,
42 SequenceGroupMetadata)
@@ -716,7 +716,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
716 context_lens: List[int] = []
717 query_lens: List[int] = []
718 prefix_block_tables: List[List[int]] = []
719 - multi_modal_inputs_list: List[MultiModalInputs] = []
719 + multi_model_kwargs_list: List[MultiModalKwargs] = []
720
721 if len(seq_group_metadata_list) == 0:
722 return PreparePromptMetadata.empty()
@@ -777,7 +777,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
777 mm_data = seq_group_metadata.multi_modal_data
778 if mm_data:
779 mm_kwargs = self.multi_modal_input_mapper(mm_data)
780 - multi_modal_inputs_list.append(mm_kwargs)
780 + multi_model_kwargs_list.append(mm_kwargs)
781
782 if seq_group_metadata.block_tables is None:
783 # During memory profiling, the block tables are not initialized
@@ -876,7 +876,7 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
876 multi_modal_placeholder_index_maps=
877 None # FIXME(kzawora): mutli-modality will not work here
878 )
879 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
879 + multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
880
881 return PreparePromptMetadata(input_tokens=input_tokens,
882 input_positions=input_positions,
vllm/worker/model_runner.py
+9 -9
@@ -38,7 +38,7 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
38 from vllm.model_executor.models import supports_lora, supports_multimodal
39 from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
40 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
41 - MultiModalInputs, MultiModalPlaceholderMap,
41 + MultiModalKwargs, MultiModalPlaceholderMap,
42 MultiModalRegistry)
43 from vllm.platforms import current_platform
44 from vllm.prompt_adapter.layers import PromptAdapterMapping
@@ -252,7 +252,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
252 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
253
254 # Multi-modal inputs.
255 - multi_modal_inputs: Optional[MultiModalInputs] = None,
255 + multi_model_kwargs: Optional[MultiModalKwargs] = None,
256 multi_modal_placeholder_maps: Optional[Dict[
257 str, MultiModalPlaceholderMap]] = None,
258
@@ -373,7 +373,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
373 prompt_adapter_prompt_mapping or [])
374
375 self.prompt_adapter_request = prompt_adapter_request
376 - self.multi_modal_inputs = multi_modal_inputs
376 + self.multi_model_kwargs = multi_model_kwargs
377 self.multi_modal_placeholder_maps = multi_modal_placeholder_maps
378 self.prefix_cache_hit = prefix_cache_hit
379
@@ -661,7 +661,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
661 mm_kwargs = self.multi_modal_input_mapper(
662 mm_data,
663 mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs)
664 - inter_data.multi_modal_inputs = mm_kwargs
664 + inter_data.multi_model_kwargs = mm_kwargs
665 inter_data.multi_modal_placeholder_maps = placeholder_maps
666
667 # special processing for mrope position deltas.
@@ -935,11 +935,11 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
935 )
936
937 # Multi-modal data.
938 - multi_modal_inputs_list = [
939 - data.multi_modal_inputs for data in self.inter_data_list
940 - if data.multi_modal_inputs is not None
938 + multi_model_kwargs_list = [
939 + data.multi_model_kwargs for data in self.inter_data_list
940 + if data.multi_model_kwargs is not None
941 ]
942 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
942 + multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
943
944 return self.model_input_cls(
945 input_tokens=input_tokens_tensor,
@@ -1649,7 +1649,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1649 kv_caches=kv_caches,
1650 attn_metadata=model_input.attn_metadata,
1651 intermediate_tensors=intermediate_tensors,
1652 - **MultiModalInputs.as_kwargs(multi_modal_kwargs,
1652 + **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
1653 device=self.device),
1654 **seqlen_agnostic_kwargs)
1655
vllm/worker/neuron_model_runner.py
+5 -5
@@ -13,7 +13,7 @@ from vllm.model_executor import SamplingMetadata
13 from vllm.model_executor.layers.sampler import SamplerOutput
14 from vllm.model_executor.model_loader.neuron import get_neuron_model
15 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16 - MultiModalInputs)
16 + MultiModalKwargs)
17 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
18 from vllm.utils import is_pin_memory_available, make_tensor_with_pad
19 from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
@@ -122,7 +122,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
122 input_block_ids: List[int] = []
123
124 seq_lens: List[int] = []
125 - multi_modal_inputs_list: List[MultiModalInputs] = []
125 + multi_model_kwargs_list: List[MultiModalKwargs] = []
126 for seq_group_metadata in seq_group_metadata_list:
127 assert seq_group_metadata.is_prompt
128 seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -149,7 +149,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
149 mm_data,
150 mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs,
151 )
152 - multi_modal_inputs_list.append(mm_kwargs)
152 + multi_model_kwargs_list.append(mm_kwargs)
153
154 max_seq_len = max(seq_lens)
155 assert max_seq_len > 0
@@ -167,7 +167,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
167 dtype=torch.long,
168 device=self.device)
169
170 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
170 + multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
171
172 return (input_tokens, input_positions, input_block_ids, seq_lens,
173 multi_modal_kwargs)
@@ -314,7 +314,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
314 input_ids=model_input.input_tokens,
315 positions=model_input.input_positions,
316 input_block_ids=model_input.input_block_ids,
317 - **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
317 + **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
318 device=self.device),
319 )
320
vllm/worker/openvino_model_runner.py
+5 -5
@@ -13,7 +13,7 @@ from vllm.model_executor import SamplingMetadata
13 from vllm.model_executor.layers.sampler import SamplerOutput
14 from vllm.model_executor.model_loader.openvino import get_model
15 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16 - MultiModalInputs, MultiModalPlaceholderMap)
16 + MultiModalKwargs, MultiModalPlaceholderMap)
17 from vllm.sequence import SequenceGroupMetadata
18 from vllm.worker.model_runner_base import ModelRunnerBase
19
@@ -102,7 +102,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
102 seq_lens: List[int] = []
103 past_lens: List[int] = []
104 query_lens: List[int] = []
105 - multi_modal_inputs_list: List[MultiModalInputs] = []
105 + multi_model_kwargs_list: List[MultiModalKwargs] = []
106 multi_modal_placeholder_maps: Dict[
107 str,
108 MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
@@ -226,7 +226,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
226 mm_data,
227 mm_processor_kwargs=seq_group_metadata.
228 mm_processor_kwargs)
229 - multi_modal_inputs_list.append(mm_kwargs)
229 + multi_model_kwargs_list.append(mm_kwargs)
230
231 for modality, placeholder_map in placeholder_maps.items():
232 multi_modal_placeholder_maps[modality].extend(
@@ -275,7 +275,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
275 multi_modal_placeholder_index_maps=placeholder_index_maps,
276 )
277
278 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
278 + multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
279
280 return ModelInput(
281 input_tokens,
@@ -341,7 +341,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
341 kv_caches,
342 "attn_metadata":
343 attn_metadata,
344 - **MultiModalInputs.as_kwargs(multi_modal_kwargs or {},
344 + **MultiModalKwargs.as_kwargs(multi_modal_kwargs or {},
345 device=self.device),
346 }
347
vllm/worker/xpu_model_runner.py
+5 -5
@@ -18,7 +18,7 @@ from vllm.model_executor import SamplingMetadataCache
18 from vllm.model_executor.layers.sampler import SamplerOutput
19 from vllm.model_executor.model_loader import get_model
20 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
21 - MultiModalInputs, MultiModalPlaceholderMap,
21 + MultiModalKwargs, MultiModalPlaceholderMap,
22 MultiModalRegistry)
23 from vllm.sampling_params import SamplingParams
24 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
@@ -160,7 +160,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
160 input_positions: List[int] = []
161 slot_mapping: List[int] = []
162 seq_lens: List[int] = []
163 - multi_modal_inputs_list: List[MultiModalInputs] = []
163 + multi_model_kwargs_list: List[MultiModalKwargs] = []
164 multi_modal_placeholder_maps: Dict[
165 str,
166 MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
@@ -192,7 +192,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
192 .from_seq_group(seq_group_metadata, positions_range)
193
194 mm_kwargs = self.runner.multi_modal_input_mapper(mm_data)
195 - multi_modal_inputs_list.append(mm_kwargs)
195 + multi_model_kwargs_list.append(mm_kwargs)
196
197 for modality, placeholder_map in placeholder_maps.items():
198 multi_modal_placeholder_maps[modality].extend(
@@ -264,7 +264,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
264 block_tables=torch.tensor([], device=self.device, dtype=torch.int),
265 )
266
267 - multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
267 + multi_modal_kwargs = MultiModalKwargs.batch(multi_model_kwargs_list)
268
269 return (input_tokens, input_positions, attn_metadata, seq_lens,
270 multi_modal_kwargs)
@@ -565,7 +565,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
565 kv_caches=kv_caches,
566 attn_metadata=model_input.attn_metadata,
567 intermediate_tensors=intermediate_tensors,
568 - **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
568 + **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
569 device=self.device))
570 # Compute the logits in the last pipeline stage.
571 if not get_pp_group().is_last_rank: