A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Refactor] Define MultiModalKwargsItems separate from MultiModalKwargs (#23053)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 18, 2025 at 17:52 UTC
27e8d1ea3ea9864f371f639daaa5315bf3250364
77 files changed
+431
-383
docs/api/README.md
+1
@@ -77,6 +77,7 @@ Internal data structures.
77
- [vllm.multimodal.inputs.MultiModalFieldElem][]
78
- [vllm.multimodal.inputs.MultiModalFieldConfig][]
79
- [vllm.multimodal.inputs.MultiModalKwargsItem][]
80
+- [vllm.multimodal.inputs.MultiModalKwargsItems][]
81
- [vllm.multimodal.inputs.MultiModalKwargs][]
82
- [vllm.multimodal.inputs.MultiModalInputs][]
83
docs/contributing/model/multimodal.md
+2
-2
@@ -629,7 +629,7 @@ Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies
629
self,
630
mm_items: MultiModalDataItems,
631
hf_processor_mm_kwargs: Mapping[str, object],
632
- out_mm_kwargs: MultiModalKwargs,
632
+ out_mm_kwargs: MultiModalKwargsItems,
633
) -> Sequence[PromptUpdate]:
634
hf_config = self.info.get_hf_config()
635
image_token_id = hf_config.image_token_index
@@ -778,7 +778,7 @@ Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies
778
self,
779
mm_items: MultiModalDataItems,
780
hf_processor_mm_kwargs: Mapping[str, object],
781
- out_mm_kwargs: MultiModalKwargs,
781
+ out_mm_kwargs: MultiModalKwargsItems,
782
) -> Sequence[PromptUpdate]:
783
hf_config = self.info.get_hf_config()
784
bos_token_id = hf_config.bos_token_id
tests/models/multimodal/processing/test_common.py
+10
-4
@@ -370,10 +370,16 @@ def _assert_inputs_equal(
370
if ignore_mm_keys is None:
371
ignore_mm_keys = set()
372
373
- assert "mm_kwargs" in a and "mm_kwargs" in b, msg
373
+ a_rest = {k: v for k, v in a.items() if k != "mm_kwargs"}
374
+ b_rest = {k: v for k, v in b.items() if k != "mm_kwargs"}
375
+
376
+ assert a_rest == b_rest, msg
377
+
378
+ a_data = a["mm_kwargs"].get_data()
379
+ b_data = b["mm_kwargs"].get_data()
380
381
for key in ignore_mm_keys:
376
- a["mm_kwargs"].pop(key, None)
377
- b["mm_kwargs"].pop(key, None)
382
+ a_data.pop(key, None)
383
+ b_data.pop(key, None)
384
379
- assert a == b, msg
385
+ assert a_data == b_data, msg
tests/models/multimodal/processing/test_glm4_1v.py
+2
-1
@@ -45,7 +45,8 @@ def test_processor_override(
45
video_token_id = tokenizer.convert_tokens_to_ids(hf_processor.video_token)
46
video_tok_count = processed_inputs["prompt_token_ids"].count(
47
video_token_id)
48
- grid_t, _, _ = processed_inputs["mm_kwargs"]["video_grid_thw"][0]
48
+ grid_t, _, _ = processed_inputs["mm_kwargs"].get_data(
49
+ )["video_grid_thw"][0]
50
51
assert grid_t == expected_grid_t
52
assert video_tok_count == expected_toks_per_frame * grid_t
tests/models/multimodal/processing/test_h2ovl.py
+2
-1
@@ -108,7 +108,8 @@ def _run_check(
108
# Ensure we have the right number of placeholders per num_crops size
109
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
110
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
111
- pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape
111
+ pixel_shape = processed_inputs["mm_kwargs"].get_data(
112
+ )["pixel_values_flat"].shape
113
114
assert img_tok_count == 256 * total_expected_num_patches
115
assert pixel_shape[0] == total_expected_num_patches
tests/models/multimodal/processing/test_internvl.py
+2
-1
@@ -68,7 +68,8 @@ def _run_check(
68
# Ensure we have the right number of placeholders per num_crops size
69
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
70
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
71
- pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape
71
+ pixel_shape = processed_inputs["mm_kwargs"].get_data(
72
+ )["pixel_values_flat"].shape
73
74
assert img_tok_count == 256 * total_expected_num_patches
75
assert pixel_shape[0] == total_expected_num_patches
tests/models/multimodal/processing/test_llama4.py
+5
-5
@@ -51,14 +51,14 @@ def test_processor_override(
51
prompt = encode_tokens(tokenizer, prompt)
52
53
processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
54
- mm_kwargs = processed_inputs["mm_kwargs"]
54
+ mm_data = processed_inputs["mm_kwargs"].get_data()
55
56
# place holder replacements
57
prompt_token_ids = processed_inputs["prompt_token_ids"]
58
assert prompt_token_ids.count(config.boi_token_index) == num_imgs
59
assert prompt_token_ids.count(config.eoi_token_index) == num_imgs
60
assert prompt_token_ids.count(vocab[hf_processor.image_token]) == num_imgs
61
- aspect_ratios = mm_kwargs["aspect_ratios"]
61
+ aspect_ratios = mm_data["aspect_ratios"]
62
num_x_separators = num_y_separators = 0
63
for tiles_y, tiles_x in aspect_ratios:
64
if tiles_x * tiles_y > 1:
@@ -80,6 +80,6 @@ def test_processor_override(
80
num_patches_per_chunk = processor.info.get_patch_per_chunk(
81
config.vision_config)
82
assert prompt_token_ids.count(config.image_token_index) \
83
- == mm_kwargs["patches_per_image"].sum() * num_patches_per_chunk
84
- assert mm_kwargs["pixel_values"].shape[0] \
85
- == mm_kwargs["patches_per_image"].sum()
83
+ == sum(mm_data["patches_per_image"]) * num_patches_per_chunk
84
+ assert len(mm_data["pixel_values"]) \
85
+ == sum(mm_data["patches_per_image"])
tests/models/multimodal/processing/test_mllama.py
+3
-3
@@ -49,18 +49,18 @@ def test_profiling(
49
encoder_seq_lens = [len(dummy_encoder_data.prompt_token_ids)
50
] * max_num_seqs
51
52
- mm_kwargs = processor.apply(
52
+ mm_data = processor.apply(
53
prompt=dummy_mm_data.prompt,
54
mm_data=dummy_mm_data.mm_data,
55
hf_processor_mm_kwargs=dict(),
56
- )["mm_kwargs"]
56
+ )["mm_kwargs"].get_data()
57
58
# Get the actual number of encoder tokens for each sample.
59
# Because attn_metadata.encoder_seq_lens only counts the last
60
# group of images for each sample, which is used to cheat the
61
# block manager to allocate blocks for those images only.
62
# See MllamaMultiModalProcessor for more details.
63
- num_tiles = [[t] for t in mm_kwargs.pop("num_tiles")]
63
+ num_tiles = [[t] for t in mm_data.pop("num_tiles")]
64
num_tokens_per_tile = calc_token_per_chunk(image_size)
65
actual_encoder_seq_lens = [
66
sum(num_tile) * num_tokens_per_tile for num_tile in num_tiles
tests/models/multimodal/processing/test_mllama4.py
+5
-5
@@ -38,21 +38,21 @@ def test_profiling(model_id: str, max_model_len: int):
38
39
hf_config = ctx.get_hf_config(Llama4Config)
40
41
- mm_kwargs = processor.apply(
41
+ mm_data = processor.apply(
42
prompt=dummy_mm_data.prompt,
43
mm_data=dummy_mm_data.mm_data,
44
hf_processor_mm_kwargs=dict(),
45
- )["mm_kwargs"]
45
+ )["mm_kwargs"].get_data()
46
47
image_size = hf_config.vision_config.image_size
48
patch_size = hf_config.vision_config.patch_size
49
downsample_ratio = int(
50
round(1.0 / (hf_config.vision_config.pixel_shuffle_ratio**2)))
51
tokens_per_patch = ((image_size // patch_size)**2) // downsample_ratio
52
- chunks_per_image = prod(mm_kwargs["patches_per_image"])
52
+ chunks_per_image = prod(mm_data["patches_per_image"])
53
total_num_patches = chunks_per_image * tokens_per_patch
54
- num_tiles = mm_kwargs["aspect_ratios"][0][0] * mm_kwargs["aspect_ratios"][
55
- 0][1] # x-y seperator tokens
54
+ num_tiles = mm_data["aspect_ratios"][0][0] * mm_data["aspect_ratios"][0][
55
+ 1] # x-y seperator tokens
56
total_tokens = total_num_patches.item() + num_tiles.item(
57
) + 3 # image start, image, image end
58
tests/models/multimodal/processing/test_nemotron_vl.py
+2
-1
@@ -70,7 +70,8 @@ def _run_check(
70
# Ensure we have the right number of placeholders per num_crops size
71
image_token_id = tokenizer.convert_tokens_to_ids("<image>")
72
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
73
- pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape
73
+ pixel_shape = processed_inputs["mm_kwargs"].get_data(
74
+ )["pixel_values_flat"].shape
75
print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape)
76
assert img_tok_count == 256 * total_expected_num_patches
77
assert pixel_shape[0] == total_expected_num_patches
tests/models/multimodal/processing/test_qwen2_vl.py
+2
-1
@@ -48,7 +48,8 @@ def test_processor_override(
48
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
49
image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
50
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
51
- pixel_shape = processed_inputs["mm_kwargs"]["pixel_values"].shape
51
+ pixel_shape = processed_inputs["mm_kwargs"].get_data(
52
+ )["pixel_values"].shape
53
54
assert img_tok_count == expected_toks_per_img * num_imgs
55
assert pixel_shape[0] == expected_pixels_shape[0] * num_imgs
tests/models/multimodal/test_tensor_schema.py
+1
-1
@@ -128,7 +128,7 @@ def create_batched_mm_kwargs(
128
)["mm_kwargs"]
129
items = [
130
item for modality in supported_mm_limits
131
- for item in mm_kwargs.get_items(modality)
131
+ for item in mm_kwargs[modality]
132
]
133
return group_mm_kwargs_by_modality(items)
134
tests/multimodal/test_cache.py
+6
-5
@@ -4,8 +4,8 @@ import pytest
4
import torch
5
6
from vllm.multimodal.cache import MultiModalCache, MultiModalCacheItemMetadata
7
-from vllm.multimodal.inputs import (MultiModalFieldElem, MultiModalKwargs,
8
- MultiModalKwargsItem,
7
+from vllm.multimodal.inputs import (MultiModalFieldElem, MultiModalKwargsItem,
8
+ MultiModalKwargsItems,
9
MultiModalSharedField)
10
11
@@ -24,8 +24,8 @@ def _dummy_item(modality: str, size_by_key: dict[str, int]):
24
])
25
26
27
-def _dummy_kw(size_by_key_modality: dict[str, dict[str, int]]):
28
- return MultiModalKwargs([
27
+def _dummy_items(size_by_key_modality: dict[str, dict[str, int]]):
28
+ return MultiModalKwargsItems.from_seq([
29
_dummy_item(modality, size_by_key)
30
for modality, size_by_key in size_by_key_modality.items()
31
])
@@ -37,7 +37,8 @@ def _dummy_kw(size_by_key_modality: dict[str, dict[str, int]]):
37
[
38
(_dummy_item("a", {"a1": 100}), 100),
39
(_dummy_item("a", {"a1": 100, "a2": 110}), 210),
40
- (_dummy_kw({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}), 460), # noqa: E501
40
+ (_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}), 460), # noqa: E501
41
+ (_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}).get_data(), 460), # noqa: E501
42
],
43
)
44
# yapf: enable
tests/v1/test_serial_utils.py
+13
-9
@@ -11,7 +11,8 @@ import torch
11
12
from vllm.multimodal.inputs import (MultiModalBatchedField,
13
MultiModalFieldElem, MultiModalFlatField,
14
- MultiModalKwargs, MultiModalKwargsItem,
14
+ MultiModalKwargsItem,
15
+ MultiModalKwargsItems,
16
MultiModalSharedField, NestedTensors)
17
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
18
@@ -96,7 +97,7 @@ def test_encode_decode(monkeypatch: pytest.MonkeyPatch):
97
98
99
class MyRequest(msgspec.Struct):
99
- mm: Optional[list[MultiModalKwargs]]
100
+ mm: Optional[list[MultiModalKwargsItems]]
101
102
103
def test_multimodal_kwargs():
@@ -119,7 +120,7 @@ def test_multimodal_kwargs():
120
audio = MultiModalKwargsItem.from_elems([e1])
121
video = MultiModalKwargsItem.from_elems([e2])
122
image = MultiModalKwargsItem.from_elems([e3, e4])
122
- mm = MultiModalKwargs([audio, video, image])
123
+ mm = MultiModalKwargsItems.from_seq([audio, video, image])
124
125
# pack mm kwargs into a mock request so that it can be decoded properly
126
req = MyRequest([mm])
@@ -133,19 +134,22 @@ def test_multimodal_kwargs():
134
135
total_len = sum(memoryview(x).cast("B").nbytes for x in encoded)
136
136
- # expected total encoding length, should be 14255, +-20 for minor changes
137
- assert 14250 <= total_len <= 14300
138
- decoded: MultiModalKwargs = decoder.decode(encoded).mm[0]
137
+ # expected total encoding length, should be 14306, +-20 for minor changes
138
+ assert 14275 <= total_len <= 14325
139
+ decoded = decoder.decode(encoded).mm[0]
140
+ assert isinstance(decoded, MultiModalKwargsItems)
141
142
# check all modalities were recovered and do some basic sanity checks
141
- assert len(decoded.modalities) == 3
142
- images = decoded.get_items("image")
143
+ assert len(decoded) == 3
144
+ images = decoded["image"]
145
assert len(images) == 1
146
assert len(images[0].items()) == 2
147
assert list(images[0].keys()) == ["i0", "i1"]
148
149
# check the tensor contents and layout in the main dict
148
- assert all(nested_equal(mm[k], decoded[k]) for k in mm)
150
+ mm_data = mm.get_data()
151
+ decoded_data = decoded.get_data()
152
+ assert all(nested_equal(mm_data[k], decoded_data[k]) for k in mm_data)
153
154
155
def nested_equal(a: NestedTensors, b: NestedTensors):
vllm/executor/msgspec_utils.py
+7
-2
@@ -4,11 +4,12 @@
4
from array import array
5
from typing import Any, Type
6
7
+from vllm.multimodal.inputs import MultiModalKwargs
8
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE
9
10
11
def encode_hook(obj: Any) -> Any:
11
- """Custom msgspec enc hook that supports array types.
12
+ """Custom msgspec enc hook that supports array types and MultiModalKwargs.
13
14
See https://jcristharif.com/msgspec/api.html#msgspec.msgpack.Encoder
15
"""
@@ -17,10 +18,12 @@ def encode_hook(obj: Any) -> Any:
18
f"vLLM array type should use '{VLLM_TOKEN_ID_ARRAY_TYPE}' type. "
19
f"Given array has a type code of {obj.typecode}.")
20
return obj.tobytes()
21
+ if isinstance(obj, MultiModalKwargs):
22
+ return dict(obj)
23
24
25
def decode_hook(type: Type, obj: Any) -> Any:
23
- """Custom msgspec dec hook that supports array types.
26
+ """Custom msgspec dec hook that supports array types and MultiModalKwargs.
27
28
See https://jcristharif.com/msgspec/api.html#msgspec.msgpack.Encoder
29
"""
@@ -28,3 +31,5 @@ def decode_hook(type: Type, obj: Any) -> Any:
31
deserialized = array(VLLM_TOKEN_ID_ARRAY_TYPE)
32
deserialized.frombytes(obj)
33
return deserialized
34
+ if type is MultiModalKwargs:
35
+ return MultiModalKwargs(obj)
vllm/model_executor/models/aria.py
+2
-2
@@ -22,7 +22,7 @@ from vllm.model_executor.model_loader.weight_utils import (
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
25
- MultiModalKwargs)
25
+ MultiModalKwargsItems)
26
from vllm.multimodal.parse import MultiModalDataItems
27
from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
BaseProcessingInfo, PromptReplacement,
@@ -470,7 +470,7 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor[AriaProcessingInfo]):
470
self,
471
mm_items: MultiModalDataItems,
472
hf_processor_mm_kwargs: Mapping[str, object],
473
- out_mm_kwargs: MultiModalKwargs,
473
+ out_mm_kwargs: MultiModalKwargsItems,
474
) -> Sequence[PromptUpdate]:
475
hf_config = self.info.get_hf_config()
476
image_token_id = hf_config.image_token_index
vllm/model_executor/models/aya_vision.py
+2
-2
@@ -18,7 +18,7 @@ from transformers.models.got_ocr2.image_processing_got_ocr2 import (
18
from vllm.config import VllmConfig
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
-from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargs
21
+from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems
22
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
23
MultiModalDataItems)
24
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -242,7 +242,7 @@ class AyaVisionMultiModalProcessor(
242
self,
243
mm_items: MultiModalDataItems,
244
hf_processor_mm_kwargs: Mapping[str, object],
245
- out_mm_kwargs: MultiModalKwargs,
245
+ out_mm_kwargs: MultiModalKwargsItems,
246
) -> Sequence[PromptUpdate]:
247
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
248
image_token = hf_processor.image_token
vllm/model_executor/models/blip2.py
+2
-2
@@ -15,7 +15,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
15
from vllm.model_executor.sampling_metadata import SamplingMetadata
16
from vllm.multimodal import MULTIMODAL_REGISTRY
17
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18
- MultiModalKwargs)
18
+ MultiModalKwargsItems)
19
from vllm.multimodal.parse import MultiModalDataItems
20
from vllm.multimodal.processing import (BaseMultiModalProcessor,
21
BaseProcessingInfo, PromptIndexTargets,
@@ -492,7 +492,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
492
self,
493
mm_items: MultiModalDataItems,
494
hf_processor_mm_kwargs: Mapping[str, object],
495
- out_mm_kwargs: MultiModalKwargs,
495
+ out_mm_kwargs: MultiModalKwargsItems,
496
) -> Sequence[PromptUpdate]:
497
tokenizer = self.info.get_tokenizer()
498
vocab = tokenizer.get_vocab()
vllm/model_executor/models/chameleon.py
+2
-2
@@ -31,7 +31,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
31
from vllm.model_executor.utils import set_weight_attrs
32
from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34
- MultiModalKwargs)
34
+ MultiModalKwargsItems)
35
from vllm.multimodal.parse import MultiModalDataItems
36
from vllm.multimodal.processing import (BaseMultiModalProcessor,
37
BaseProcessingInfo, PromptReplacement,
@@ -151,7 +151,7 @@ class ChameleonMultiModalProcessor(
151
self,
152
mm_items: MultiModalDataItems,
153
hf_processor_mm_kwargs: Mapping[str, object],
154
- out_mm_kwargs: MultiModalKwargs,
154
+ out_mm_kwargs: MultiModalKwargsItems,
155
) -> Sequence[PromptUpdate]:
156
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
157
tokenizer = self.info.get_tokenizer()
vllm/model_executor/models/cohere2_vision.py
+2
-2
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
21
from vllm.model_executor.layers.quantization.awq import AWQConfig
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
-from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargs
24
+from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems
25
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
26
MultiModalDataItems)
27
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -241,7 +241,7 @@ class Cohere2VisionMultiModalProcessor(
241
self,
242
mm_items: MultiModalDataItems,
243
hf_processor_mm_kwargs: Mapping[str, object],
244
- out_mm_kwargs: MultiModalKwargs,
244
+ out_mm_kwargs: MultiModalKwargsItems,
245
) -> Sequence[PromptUpdate]:
246
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
247
image_token = hf_processor.image_token
vllm/model_executor/models/deepseek_vl2.py
+4
-3
@@ -21,7 +21,7 @@ from vllm.model_executor.model_loader.utils import set_default_torch_dtype
21
from vllm.model_executor.models.transformers import replace_linear_class
22
from vllm.multimodal import MULTIMODAL_REGISTRY
23
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24
- MultiModalKwargs, NestedTensors)
24
+ MultiModalKwargsItems, NestedTensors)
25
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
26
ImageSize, MultiModalDataItems)
27
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -252,7 +252,7 @@ class DeepseekVL2MultiModalProcessor(
252
self,
253
mm_items: MultiModalDataItems,
254
hf_processor_mm_kwargs: Mapping[str, object],
255
- out_mm_kwargs: MultiModalKwargs,
255
+ out_mm_kwargs: MultiModalKwargsItems,
256
) -> Sequence[PromptUpdate]:
257
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
258
@@ -291,7 +291,8 @@ class DeepseekVL2MultiModalProcessor(
291
tokenization_kwargs: Mapping[str, object],
292
*,
293
return_mm_hashes: bool,
294
- ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
294
+ ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
295
+ bool]:
296
# The processor logic is different for len(images) <= 2 vs > 2
297
# Since the processing cache assumes that the processor output is
298
# invariant of how many images are passed per prompt, we only
vllm/model_executor/models/florence2.py
+2
-2
@@ -21,7 +21,7 @@ from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.multimodal import MULTIMODAL_REGISTRY
23
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24
- MultiModalKwargs)
24
+ MultiModalKwargsItems)
25
from vllm.multimodal.parse import MultiModalDataItems
26
from vllm.multimodal.processing import (BaseProcessingInfo,
27
EncDecMultiModalProcessor,
@@ -860,7 +860,7 @@ class Florence2MultiModalProcessor(
860
self,
861
mm_items: MultiModalDataItems,
862
hf_processor_mm_kwargs: Mapping[str, object],
863
- out_mm_kwargs: MultiModalKwargs,
863
+ out_mm_kwargs: MultiModalKwargsItems,
864
) -> Sequence[PromptUpdate]:
865
hf_config = self.info.get_hf_config()
866
pad_token_id = hf_config.pad_token_id
vllm/model_executor/models/fuyu.py
+2
-2
@@ -32,7 +32,7 @@ from vllm.model_executor.models.persimmon import PersimmonForCausalLM
32
from vllm.model_executor.sampling_metadata import SamplingMetadata
33
from vllm.multimodal import MULTIMODAL_REGISTRY
34
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35
- MultiModalKwargs)
35
+ MultiModalKwargsItems)
36
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
37
MultiModalDataItems)
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -226,7 +226,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
226
self,
227
mm_items: MultiModalDataItems,
228
hf_processor_mm_kwargs: Mapping[str, object],
229
- out_mm_kwargs: MultiModalKwargs,
229
+ out_mm_kwargs: MultiModalKwargsItems,
230
) -> Sequence[PromptUpdate]:
231
hf_config = self.info.get_hf_config()
232
bos_token_id = hf_config.bos_token_id
vllm/model_executor/models/gemma3_mm.py
+2
-2
@@ -17,7 +17,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
17
from vllm.model_executor.sampling_metadata import SamplingMetadata
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
20
- MultiModalKwargs)
20
+ MultiModalKwargsItems)
21
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
22
MultiModalDataItems)
23
# yapf: disable
@@ -311,7 +311,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
311
self,
312
mm_items: MultiModalDataItems,
313
hf_processor_mm_kwargs: Mapping[str, Any],
314
- out_mm_kwargs: MultiModalKwargs,
314
+ out_mm_kwargs: MultiModalKwargsItems,
315
) -> Sequence[PromptUpdate]:
316
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
317
image_token = hf_processor.boi_token
vllm/model_executor/models/gemma3n_mm.py
+2
-2
@@ -24,7 +24,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
from vllm.multimodal import MULTIMODAL_REGISTRY
26
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
27
- MultiModalKwargs)
27
+ MultiModalKwargsItems)
28
from vllm.multimodal.parse import (ImageProcessorItems, MultiModalDataItems,
29
MultiModalDataParser)
30
# yapf: disable
@@ -209,7 +209,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]
209
self,
210
mm_items: MultiModalDataItems,
211
hf_processor_mm_kwargs: Mapping[str, Any],
212
- out_mm_kwargs: MultiModalKwargs,
212
+ out_mm_kwargs: MultiModalKwargsItems,
213
) -> Sequence[PromptUpdate]:
214
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
215
vllm/model_executor/models/glm4_1v.py
+6
-4
@@ -59,7 +59,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
59
from vllm.model_executor.models.module_mapping import MultiModelKeys
60
from vllm.multimodal import MULTIMODAL_REGISTRY
61
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
62
- MultiModalKwargs, VideoItem)
62
+ MultiModalKwargsItems, VideoItem)
63
from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
64
MultiModalDataParser)
65
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -1158,7 +1158,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1158
self,
1159
mm_items: MultiModalDataItems,
1160
hf_processor_mm_kwargs: Mapping[str, Any],
1161
- out_mm_kwargs: MultiModalKwargs,
1161
+ out_mm_kwargs: MultiModalKwargsItems,
1162
) -> Sequence[PromptUpdate]:
1163
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1164
image_processor = self.info.get_image_processor(
@@ -1175,14 +1175,16 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1175
merge_length = image_processor.merge_size**2
1176
1177
def get_image_replacement_glm4v(item_idx: int):
1178
- grid_thw = out_mm_kwargs["image_grid_thw"][item_idx]
1178
+ out_item = out_mm_kwargs["image"][item_idx]
1179
+ grid_thw = out_item["image_grid_thw"].data
1180
assert isinstance(grid_thw, torch.Tensor)
1181
1182
num_tokens = int(grid_thw.prod()) // merge_length
1183
return [hf_processor.image_token_id] * num_tokens
1184
1185
def get_video_replacement_glm4v(item_idx: int):
1185
- grid_thw = out_mm_kwargs["video_grid_thw"][item_idx]
1186
+ out_item = out_mm_kwargs["video"][item_idx]
1187
+ grid_thw = out_item["video_grid_thw"].data
1188
assert isinstance(grid_thw, torch.Tensor)
1189
1190
video, metadata = mm_items["video"][item_idx]
vllm/model_executor/models/glm4v.py
+2
-2
@@ -30,7 +30,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
30
from vllm.model_executor.models.module_mapping import MultiModelKeys
31
from vllm.multimodal import MULTIMODAL_REGISTRY
32
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
33
- MultiModalKwargs)
33
+ MultiModalKwargsItems)
34
from vllm.multimodal.parse import MultiModalDataItems
35
from vllm.multimodal.processing import (BaseMultiModalProcessor,
36
BaseProcessingInfo, PromptReplacement,
@@ -503,7 +503,7 @@ class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
503
self,
504
mm_items: MultiModalDataItems,
505
hf_processor_mm_kwargs: Mapping[str, object],
506
- out_mm_kwargs: MultiModalKwargs,
506
+ out_mm_kwargs: MultiModalKwargsItems,
507
) -> Sequence[PromptUpdate]:
508
hf_config = self.info.get_hf_config()
509
vllm/model_executor/models/granite_speech.py
+2
-2
@@ -40,7 +40,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
from vllm.multimodal import MULTIMODAL_REGISTRY
42
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
43
- MultiModalKwargs)
43
+ MultiModalKwargsItems)
44
from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
45
MultiModalDataParser)
46
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -118,7 +118,7 @@ class GraniteSpeechMultiModalProcessor(
118
self,
119
mm_items: MultiModalDataItems,
120
hf_processor_mm_kwargs: Mapping[str, object],
121
- out_mm_kwargs: MultiModalKwargs,
121
+ out_mm_kwargs: MultiModalKwargsItems,
122
) -> list[PromptUpdate]:
123
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
124
tokenizer = self.info.get_tokenizer()
vllm/model_executor/models/h2ovl.py
+9
-7
@@ -17,7 +17,7 @@ from transformers import PretrainedConfig
17
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
-from vllm.multimodal.inputs import MultiModalKwargs
20
+from vllm.multimodal.inputs import MultiModalKwargsItems
21
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
22
MultiModalDataItems)
23
from vllm.multimodal.processing import (MultiModalHashes, PromptReplacement,
@@ -425,18 +425,19 @@ class H2OVLMultiModalProcessor(
425
self,
426
mm_items: MultiModalDataItems,
427
hf_processor_mm_kwargs: Mapping[str, object],
428
- out_mm_kwargs: MultiModalKwargs,
428
+ out_mm_kwargs: MultiModalKwargsItems,
429
) -> Sequence[PromptUpdate]:
430
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
431
432
- if "image_num_patches" in out_mm_kwargs:
433
- image_num_patches = out_mm_kwargs["image_num_patches"]
432
+ out_mm_data = out_mm_kwargs.get_data()
433
+ if "image_num_patches" in out_mm_data:
434
+ image_num_patches = out_mm_data["image_num_patches"]
435
assert isinstance(image_num_patches, torch.Tensor)
436
image_num_patches = image_num_patches.tolist()
436
- elif "image_embeds" in out_mm_kwargs:
437
+ elif "image_embeds" in out_mm_data:
438
# TODO: Use image size information in dictionary embedding inputs
439
# to compute num_patches (similar to Qwen2-VL)
439
- image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
440
+ image_num_patches = [None] * len(out_mm_data["image_embeds"])
441
else:
442
image_num_patches = []
443
@@ -479,7 +480,8 @@ class H2OVLMultiModalProcessor(
480
tokenization_kwargs: Mapping[str, object],
481
*,
482
return_mm_hashes: bool,
482
- ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
483
+ ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
484
+ bool]:
485
# The processor logic is different for len(images) <= 1 vs > 1
486
# Since the processing cache assumes that the processor output is
487
# invariant of how many images are passed per prompt, we only
vllm/model_executor/models/hyperclovax_vision.py
+14
-13
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
34
from vllm.model_executor.sampling_metadata import SamplingMetadata
35
from vllm.multimodal import MULTIMODAL_REGISTRY
36
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
37
- MultiModalKwargs)
37
+ MultiModalKwargsItems)
38
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
39
from vllm.multimodal.processing import (BaseMultiModalProcessor,
40
BaseProcessingInfo, ProcessingCache,
@@ -295,7 +295,7 @@ class HCXVisionMultiModalProcessor(
295
self,
296
mm_items: MultiModalDataItems,
297
hf_processor_mm_kwargs: Mapping[str, object],
298
- out_mm_kwargs: MultiModalKwargs,
298
+ out_mm_kwargs: MultiModalKwargsItems,
299
) -> Sequence[PromptUpdate]:
300
hf_config = self.info.get_hf_config()
301
placeholder = {
@@ -306,21 +306,22 @@ class HCXVisionMultiModalProcessor(
306
def get_replacement_hyperclovax(
307
item_idx: int,
308
modality: str,
309
- out_mm_kwargs: MultiModalKwargs,
309
+ out_mm_kwargs: MultiModalKwargsItems,
310
):
311
- num_tokens = None
311
+ out_item = out_mm_kwargs[modality][item_idx]
312
+
313
if modality == "image":
314
+ lens = out_item["vision_query_lengths_images"].data
315
num_tokens = self.info.get_num_image_tokens(
314
- vision_query_length=out_mm_kwargs[
315
- "vision_query_lengths_images"][item_idx], )
316
- if modality == "video":
316
+ vision_query_length=lens)
317
+ elif modality == "video":
318
+ lens = out_item["vision_query_lengths_videos"].data
319
num_tokens = self.info.get_num_video_tokens(
318
- vision_query_length=out_mm_kwargs[
319
- "vision_query_lengths_videos"][item_idx], )
320
- assert isinstance(num_tokens, int)
321
- return [
322
- placeholder[modality],
323
- ] * num_tokens
320
+ vision_query_length=lens)
321
+ else:
322
+ raise NotImplementedError(modality)
323
+
324
+ return [placeholder[modality]] * num_tokens
325
326
return [
327
PromptReplacement(
vllm/model_executor/models/idefics3.py
+2
-2
@@ -34,7 +34,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
34
from vllm.model_executor.sampling_metadata import SamplingMetadata
35
from vllm.multimodal import MULTIMODAL_REGISTRY
36
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
37
- MultiModalKwargs)
37
+ MultiModalKwargsItems)
38
from vllm.multimodal.parse import ImageProcessorItems, ImageSize
39
# yapf conflicts with isort for this block
40
# yapf: disable
@@ -374,7 +374,7 @@ class Idefics3MultiModalProcessor(
374
self,
375
mm_items: MultiModalDataItems,
376
hf_processor_mm_kwargs: Mapping[str, object],
377
- out_mm_kwargs: MultiModalKwargs,
377
+ out_mm_kwargs: MultiModalKwargsItems,
378
) -> Sequence[PromptUpdate]:
379
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
380
image_token, _, _ = self.info._get_image_token(hf_processor)
vllm/model_executor/models/interns1.py
+7
-6
@@ -24,7 +24,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
from vllm.multimodal import MULTIMODAL_REGISTRY
26
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
27
- MultiModalKwargs, NestedTensors)
27
+ MultiModalKwargsItems, NestedTensors)
28
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
29
ImageSize, MultiModalDataItems)
30
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -399,7 +399,7 @@ class InternS1MultiModalProcessor(
399
self,
400
mm_items: MultiModalDataItems,
401
hf_processor_mm_kwargs: Mapping[str, object],
402
- out_mm_kwargs: MultiModalKwargs,
402
+ out_mm_kwargs: MultiModalKwargsItems,
403
) -> Sequence[PromptUpdate]:
404
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
405
img_context_token = hf_processor.image_token
@@ -407,15 +407,16 @@ class InternS1MultiModalProcessor(
407
end_image_token = hf_processor.end_image_token
408
video_token = hf_processor.video_token
409
410
- if "video_num_patches" in out_mm_kwargs:
411
- video_num_patches = out_mm_kwargs["video_num_patches"]
410
+ out_mm_data = out_mm_kwargs.get_data()
411
+ if "video_num_patches" in out_mm_data:
412
+ video_num_patches = out_mm_data["video_num_patches"]
413
assert isinstance(video_num_patches, torch.Tensor)
414
video_num_patches = video_num_patches.tolist()
415
else:
416
video_num_patches = []
417
417
- if "image_num_patches" in out_mm_kwargs:
418
- image_num_patches = out_mm_kwargs["image_num_patches"]
418
+ if "image_num_patches" in out_mm_data:
419
+ image_num_patches = out_mm_data["image_num_patches"]
420
assert isinstance(image_num_patches, torch.Tensor)
421
image_num_patches = image_num_patches.tolist()
422
else:
vllm/model_executor/models/internvl.py
+21
-13
@@ -28,7 +28,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
28
from vllm.multimodal import MULTIMODAL_REGISTRY
29
from vllm.multimodal.image import convert_image_mode
30
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
31
- MultiModalKwargs, NestedTensors)
31
+ MultiModalKwargsItems, NestedTensors)
32
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
33
ImageSize, MultiModalDataItems)
34
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -797,18 +797,19 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
797
self,
798
mm_items: MultiModalDataItems,
799
hf_processor_mm_kwargs: Mapping[str, object],
800
- out_mm_kwargs: MultiModalKwargs,
800
+ out_mm_kwargs: MultiModalKwargsItems,
801
) -> Sequence[PromptUpdate]:
802
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
803
804
- if "image_num_patches" in out_mm_kwargs:
805
- image_num_patches = out_mm_kwargs["image_num_patches"]
804
+ out_mm_data = out_mm_kwargs.get_data()
805
+ if "image_num_patches" in out_mm_data:
806
+ image_num_patches = out_mm_data["image_num_patches"]
807
assert isinstance(image_num_patches, torch.Tensor)
808
image_num_patches = image_num_patches.tolist()
808
- elif "image_embeds" in out_mm_kwargs:
809
+ elif "image_embeds" in out_mm_data:
810
# TODO: Use image size information in dictionary embedding inputs
811
# to compute num_patches (similar to Qwen2-VL)
811
- image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
812
+ image_num_patches = [None] * len(out_mm_data["image_embeds"])
813
else:
814
image_num_patches = []
815
@@ -966,15 +967,19 @@ class InternVLMultiModalProcessor(
967
self,
968
mm_items: MultiModalDataItems,
969
hf_processor_mm_kwargs: Mapping[str, object],
969
- out_mm_kwargs: MultiModalKwargs,
970
+ out_mm_kwargs: MultiModalKwargsItems,
971
) -> Sequence[PromptUpdate]:
971
- prompt_repl: list[PromptUpdate] = super()._get_prompt_updates(
972
- mm_items, hf_processor_mm_kwargs, out_mm_kwargs)
972
+ prompt_repl = super()._get_prompt_updates(
973
+ mm_items=mm_items,
974
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
975
+ out_mm_kwargs=out_mm_kwargs,
976
+ )
977
978
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
979
976
- if "video_num_patches" in out_mm_kwargs:
977
- video_num_patches = out_mm_kwargs["video_num_patches"]
980
+ out_mm_data = out_mm_kwargs.get_data()
981
+ if "video_num_patches" in out_mm_data:
982
+ video_num_patches = out_mm_data["video_num_patches"]
983
assert isinstance(video_num_patches, torch.Tensor)
984
video_num_patches = video_num_patches.tolist()
985
else:
@@ -992,12 +997,15 @@ class InternVLMultiModalProcessor(
997
video_context_token=hf_processor.video_token)
998
999
if self.info.supports_video:
995
- prompt_repl.append(
1000
+ prompt_repl = [
1001
+ *prompt_repl,
1002
PromptReplacement(
1003
modality="video",
1004
target="<video>",
1005
replacement=get_video_replacement_internvl,
1000
- ))
1006
+ )
1007
+ ]
1008
+
1009
return prompt_repl
1010
1011
vllm/model_executor/models/keye.py
+4
-3
@@ -33,7 +33,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
33
from vllm.multimodal import MULTIMODAL_REGISTRY, NestedTensors
34
from vllm.multimodal.inputs import (ImageItem, ModalityData,
35
MultiModalDataDict, MultiModalFieldConfig,
36
- MultiModalKwargs, VideoItem)
36
+ MultiModalKwargsItems, VideoItem)
37
from vllm.multimodal.parse import (DictEmbeddingItems, ImageSize,
38
ModalityDataItems, MultiModalDataItems,
39
MultiModalDataParser)
@@ -1192,7 +1192,7 @@ class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
1192
self,
1193
mm_items: MultiModalDataItems,
1194
hf_processor_mm_kwargs: Mapping[str, Any],
1195
- out_mm_kwargs: MultiModalKwargs,
1195
+ out_mm_kwargs: MultiModalKwargsItems,
1196
) -> Sequence[PromptUpdate]:
1197
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1198
image_processor = self.info.get_image_processor(
@@ -1208,7 +1208,8 @@ class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
1208
merge_length = image_processor.merge_size**2
1209
1210
def get_replacement_keye(item_idx: int, modality: str):
1211
- grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
1211
+ out_item = out_mm_kwargs[modality][item_idx]
1212
+ grid_thw = out_item[f"{modality}_grid_thw"].data
1213
assert isinstance(grid_thw, torch.Tensor)
1214
1215
num_tokens = int(grid_thw.prod()) // merge_length
vllm/model_executor/models/kimi_vl.py
+2
-2
@@ -69,7 +69,7 @@ from vllm.model_executor.models.utils import merge_multimodal_embeddings
69
from vllm.model_executor.sampling_metadata import SamplingMetadata
70
from vllm.multimodal import MULTIMODAL_REGISTRY
71
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
72
- MultiModalKwargs, NestedTensors)
72
+ MultiModalKwargsItems, NestedTensors)
73
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
74
MultiModalDataItems)
75
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -239,7 +239,7 @@ class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
239
self,
240
mm_items: MultiModalDataItems,
241
hf_processor_mm_kwargs: Mapping[str, Any],
242
- out_mm_kwargs: MultiModalKwargs,
242
+ out_mm_kwargs: MultiModalKwargsItems,
243
) -> Sequence[PromptUpdate]:
244
image_token_id = self.info.image_token_id
245
vllm/model_executor/models/llava.py
+3
-3
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
26
- MultiModalInputs, MultiModalKwargs)
26
+ MultiModalInputs, MultiModalKwargsItems)
27
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
28
ImageSize, MultiModalDataItems)
29
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -250,7 +250,7 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]):
250
self,
251
mm_items: MultiModalDataItems,
252
hf_processor_mm_kwargs: Mapping[str, object],
253
- out_mm_kwargs: MultiModalKwargs,
253
+ out_mm_kwargs: MultiModalKwargsItems,
254
) -> Sequence[PromptUpdate]:
255
hf_config = self.info.get_hf_config()
256
image_token_id = hf_config.image_token_index
@@ -343,7 +343,7 @@ class PixtralHFMultiModalProcessor(
343
self,
344
mm_items: MultiModalDataItems,
345
hf_processor_mm_kwargs: Mapping[str, object],
346
- out_mm_kwargs: MultiModalKwargs,
346
+ out_mm_kwargs: MultiModalKwargsItems,
347
) -> Sequence[PromptUpdate]:
348
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
349
hf_config = self.info.get_hf_config()
vllm/model_executor/models/llava_next_video.py
+2
-2
@@ -16,7 +16,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
19
- MultiModalKwargs)
19
+ MultiModalKwargsItems)
20
from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
21
VideoEmbeddingItems, VideoProcessorItems)
22
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -185,7 +185,7 @@ class LlavaNextVideoMultiModalProcessor(
185
self,
186
mm_items: MultiModalDataItems,
187
hf_processor_mm_kwargs: Mapping[str, object],
188
- out_mm_kwargs: MultiModalKwargs,
188
+ out_mm_kwargs: MultiModalKwargsItems,
189
) -> Sequence[PromptUpdate]:
190
hf_config = self.info.get_hf_config()
191
video_token_id = hf_config.video_token_index
vllm/model_executor/models/llava_onevision.py
+2
-2
@@ -18,7 +18,7 @@ from vllm.model_executor.layers.activation import get_act_fn
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
21
- MultiModalKwargs)
21
+ MultiModalKwargsItems)
22
from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
23
VideoEmbeddingItems, VideoProcessorItems)
24
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
@@ -372,7 +372,7 @@ class LlavaOnevisionMultiModalProcessor(
372
self,
373
mm_items: MultiModalDataItems,
374
hf_processor_mm_kwargs: Mapping[str, object],
375
- out_mm_kwargs: MultiModalKwargs,
375
+ out_mm_kwargs: MultiModalKwargsItems,
376
) -> Sequence[PromptUpdate]:
377
image_repls = super()._get_prompt_updates(
378
mm_items=mm_items,
vllm/model_executor/models/minicpmo.py
+2
-2
@@ -40,7 +40,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
40
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
41
from vllm.model_executor.layers.quantization.gptq_marlin import (
42
GPTQMarlinConfig)
43
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
43
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
44
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
45
NestedTensors)
46
from vllm.multimodal.parse import (AudioItem, AudioProcessorItems,
@@ -316,7 +316,7 @@ class MiniCPMOMultiModalProcessor(
316
self,
317
mm_items: MultiModalDataItems,
318
hf_processor_mm_kwargs: Mapping[str, object],
319
- out_mm_kwargs: MultiModalKwargs,
319
+ out_mm_kwargs: MultiModalKwargsItems,
320
) -> Sequence[PromptUpdate]:
321
base_updates = super()._get_prompt_updates(
322
mm_items=mm_items,
vllm/model_executor/models/minicpmv.py
+2
-2
@@ -48,7 +48,7 @@ from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
48
from vllm.model_executor.models.module_mapping import MultiModelKeys
49
from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
50
from vllm.model_executor.sampling_metadata import SamplingMetadata
51
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
51
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
52
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
53
NestedTensors)
54
from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
@@ -694,7 +694,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
694
self,
695
mm_items: MultiModalDataItems,
696
hf_processor_mm_kwargs: Mapping[str, object],
697
- out_mm_kwargs: MultiModalKwargs,
697
+ out_mm_kwargs: MultiModalKwargsItems,
698
) -> Sequence[PromptUpdate]:
699
placeholders = [("image", self.info.image_pattern),
700
("video", self.info.video_pattern)]
vllm/model_executor/models/mistral3.py
+2
-2
@@ -23,7 +23,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
26
- MultiModalKwargs)
26
+ MultiModalKwargsItems)
27
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
28
MultiModalDataItems)
29
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -265,7 +265,7 @@ class Mistral3MultiModalProcessor(
265
self,
266
mm_items: MultiModalDataItems,
267
hf_processor_mm_kwargs: Mapping[str, object],
268
- out_mm_kwargs: MultiModalKwargs,
268
+ out_mm_kwargs: MultiModalKwargsItems,
269
) -> Sequence[PromptUpdate]:
270
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
271
hf_config = self.info.get_hf_config()
vllm/model_executor/models/mllama.py
+4
-3
@@ -56,7 +56,8 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
56
from vllm.model_executor.sampling_metadata import SamplingMetadata
57
from vllm.multimodal import MULTIMODAL_REGISTRY
58
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
59
- MultiModalFieldConfig, MultiModalKwargs)
59
+ MultiModalFieldConfig,
60
+ MultiModalKwargsItems)
61
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
62
MultiModalDataItems)
63
from vllm.multimodal.processing import (BaseProcessingInfo,
@@ -217,7 +218,7 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
218
# Set encoder prompt length based on the number of tiles.
219
# This tells the block manager to allocate correct number
220
# of slots for encoder tokens.
220
- num_tiles = mm_inputs["mm_kwargs"]["num_tiles"]
221
+ num_tiles = mm_inputs["mm_kwargs"].get_data()["num_tiles"]
222
decode_tiles = num_tiles[num_encode_images:num_images].sum().item()
223
num_tokens = decode_tiles * token_per_chunk
224
mm_inputs["encoder_prompt_token_ids"] = [image_token_id
@@ -302,7 +303,7 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
303
self,
304
mm_items: MultiModalDataItems,
305
hf_processor_mm_kwargs: Mapping[str, object],
305
- out_mm_kwargs: MultiModalKwargs,
306
+ out_mm_kwargs: MultiModalKwargsItems,
307
) -> Sequence[PromptUpdate]:
308
token_per_chunk = self.info.get_token_per_chunk_from_config()
309
image_token_id = self.info.get_hf_config().image_token_index
vllm/model_executor/models/mllama4.py
+4
-8
@@ -44,7 +44,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.multimodal import MULTIMODAL_REGISTRY
46
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
47
- MultiModalKwargs, NestedTensors)
47
+ MultiModalKwargsItems, NestedTensors)
48
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
49
MultiModalDataItems)
50
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -646,13 +646,8 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]
646
self,
647
mm_items: MultiModalDataItems,
648
hf_processor_mm_kwargs: Mapping[str, object],
649
- out_mm_kwargs: MultiModalKwargs,
649
+ out_mm_kwargs: MultiModalKwargsItems,
650
) -> list[PromptUpdate]:
651
- assert (
652
- mm_items.get_count("image", strict=False) == 0
653
- or "aspect_ratios" in out_mm_kwargs
654
- ), "Transformers expect to include aspect_ratios in out_mm_kwargs"
655
-
651
config = self.info.get_hf_config()
652
vision_config = config.vision_config
653
@@ -662,7 +657,8 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]
657
img_patch_token = hf_processor.img_patch_token
658
659
def get_replacement(item_idx: int):
665
- aspect_ratio = out_mm_kwargs["aspect_ratios"][item_idx]
660
+ out_item = out_mm_kwargs["image"][item_idx]
661
+ aspect_ratio = out_item["aspect_ratios"].data
662
663
repl = hf_processor._prompt_split_image(
664
aspect_ratio=aspect_ratio,
vllm/model_executor/models/molmo.py
+2
-2
@@ -42,7 +42,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42
from vllm.model_executor.models.module_mapping import MultiModelKeys
43
from vllm.multimodal import MULTIMODAL_REGISTRY
44
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
45
- MultiModalKwargs)
45
+ MultiModalKwargsItems)
46
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
47
MultiModalDataItems)
48
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -1282,7 +1282,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1282
self,
1283
mm_items: MultiModalDataItems,
1284
hf_processor_mm_kwargs: Mapping[str, object],
1285
- out_mm_kwargs: MultiModalKwargs,
1285
+ out_mm_kwargs: MultiModalKwargsItems,
1286
) -> Sequence[PromptUpdate]:
1287
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1288
vllm/model_executor/models/nvlm_d.py
+7
-6
@@ -16,7 +16,7 @@ from transformers import PretrainedConfig
16
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
-from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargs
19
+from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems
20
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
21
MultiModalDataItems)
22
from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,
@@ -106,18 +106,19 @@ class NVLMMultiModalProcessor(
106
self,
107
mm_items: MultiModalDataItems,
108
hf_processor_mm_kwargs: Mapping[str, object],
109
- out_mm_kwargs: MultiModalKwargs,
109
+ out_mm_kwargs: MultiModalKwargsItems,
110
) -> Sequence[PromptUpdate]:
111
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
112
113
- if "image_num_patches" in out_mm_kwargs:
114
- image_num_patches = out_mm_kwargs["image_num_patches"]
113
+ out_mm_data = out_mm_kwargs.get_data()
114
+ if "image_num_patches" in out_mm_data:
115
+ image_num_patches = out_mm_data["image_num_patches"]
116
assert isinstance(image_num_patches, torch.Tensor)
117
image_num_patches = image_num_patches.tolist()
117
- elif "image_embeds" in out_mm_kwargs:
118
+ elif "image_embeds" in out_mm_data:
119
# TODO: Use image size information in dictionary embedding inputs
120
# to compute num_patches (similar to Qwen2-VL)
120
- image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
121
+ image_num_patches = [None] * len(out_mm_data["image_embeds"])
122
else:
123
image_num_patches = []
124
vllm/model_executor/models/ovis.py
+5
-4
@@ -42,7 +42,7 @@ from vllm.model_executor.models.utils import (AutoWeightsLoader, flatten_bn,
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
43
from vllm.multimodal import MULTIMODAL_REGISTRY
44
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
45
- MultiModalKwargs)
45
+ MultiModalKwargsItems)
46
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
47
from vllm.multimodal.processing import (BaseMultiModalProcessor,
48
BaseProcessingInfo, PromptReplacement)
@@ -375,11 +375,12 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
375
self,
376
mm_items: MultiModalDataItems,
377
hf_processor_mm_kwargs: Mapping[str, object],
378
- out_mm_kwargs: MultiModalKwargs,
378
+ out_mm_kwargs: MultiModalKwargsItems,
379
) -> list[PromptReplacement]:
380
381
- def get_replacement_ovis(item_idx):
382
- grid = out_mm_kwargs["grids"][item_idx]
381
+ def get_replacement_ovis(item_idx: int):
382
+ out_item = out_mm_kwargs["image"][item_idx]
383
+ grid = out_item["grids"].data
384
385
hf_processor = self.info.get_hf_processor()
386
return hf_processor.construct_image_placeholders(grid)
vllm/model_executor/models/paligemma.py
+2
-2
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
12
from vllm.model_executor.sampling_metadata import SamplingMetadata
13
from vllm.multimodal import MULTIMODAL_REGISTRY
14
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
15
- MultiModalInputs, MultiModalKwargs)
15
+ MultiModalInputs, MultiModalKwargsItems)
16
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
17
MultiModalDataItems)
18
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -146,7 +146,7 @@ class PaliGemmaMultiModalProcessor(
146
self,
147
mm_items: MultiModalDataItems,
148
hf_processor_mm_kwargs: Mapping[str, object],
149
- out_mm_kwargs: MultiModalKwargs,
149
+ out_mm_kwargs: MultiModalKwargsItems,
150
) -> Sequence[PromptUpdate]:
151
hf_config = self.info.get_hf_config()
152
image_token_id = hf_config.image_token_index
vllm/model_executor/models/phi3v.py
+2
-2
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32
from vllm.model_executor.sampling_metadata import SamplingMetadata
33
from vllm.multimodal import MULTIMODAL_REGISTRY
34
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35
- MultiModalKwargs)
35
+ MultiModalKwargsItems)
36
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
37
ImageSize, MultiModalDataItems)
38
# yapf conflicts with isort for this block
@@ -410,7 +410,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
410
self,
411
mm_items: MultiModalDataItems,
412
hf_processor_mm_kwargs: Mapping[str, Any],
413
- out_mm_kwargs: MultiModalKwargs,
413
+ out_mm_kwargs: MultiModalKwargsItems,
414
) -> Sequence[PromptUpdate]:
415
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
416
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
vllm/model_executor/models/phi4_multimodal.py
+2
-2
@@ -30,7 +30,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
30
from vllm.model_executor.sampling_metadata import SamplingMetadata
31
from vllm.multimodal import MULTIMODAL_REGISTRY
32
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
33
- MultiModalKwargs, NestedTensors)
33
+ MultiModalKwargsItems, NestedTensors)
34
from vllm.multimodal.parse import (AudioProcessorItems, ImageEmbeddingItems,
35
ImageProcessorItems, ImageSize,
36
MultiModalDataItems, MultiModalDataParser)
@@ -1029,7 +1029,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
1029
self,
1030
mm_items: MultiModalDataItems,
1031
hf_processor_mm_kwargs: Mapping[str, Any],
1032
- out_mm_kwargs: MultiModalKwargs,
1032
+ out_mm_kwargs: MultiModalKwargsItems,
1033
) -> Sequence[PromptUpdate]:
1034
tokenizer = self.info.get_tokenizer()
1035
image_token_id = tokenizer.vocab[tokenizer.image_token]
vllm/model_executor/models/phi4mm.py
+2
-2
@@ -21,7 +21,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.multimodal import MULTIMODAL_REGISTRY
23
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24
- MultiModalKwargs, NestedTensors)
24
+ MultiModalKwargsItems, NestedTensors)
25
from vllm.multimodal.parse import (AudioProcessorItems, ImageEmbeddingItems,
26
ImageProcessorItems, ImageSize,
27
MultiModalDataItems, MultiModalDataParser)
@@ -802,7 +802,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
802
self,
803
mm_items: MultiModalDataItems,
804
hf_processor_mm_kwargs: Mapping[str, Any],
805
- out_mm_kwargs: MultiModalKwargs,
805
+ out_mm_kwargs: MultiModalKwargsItems,
806
) -> Sequence[PromptUpdate]:
807
image_tokens: list[str] = self.info.image_tokens # type: ignore
808
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
vllm/model_executor/models/pixtral.py
+4
-3
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35
from vllm.model_executor.sampling_metadata import SamplingMetadata
36
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
36
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
37
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
38
NestedTensors)
39
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
@@ -273,7 +273,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
273
self,
274
mm_items: MultiModalDataItems,
275
hf_processor_mm_kwargs: Mapping[str, object],
276
- out_mm_kwargs: MultiModalKwargs,
276
+ out_mm_kwargs: MultiModalKwargsItems,
277
) -> Sequence[PromptUpdate]:
278
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
279
@@ -309,7 +309,8 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
309
tokenization_kwargs: Mapping[str, object],
310
*,
311
return_mm_hashes: bool,
312
- ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
312
+ ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
313
+ bool]:
314
(
315
prompt_ids,
316
mm_kwargs,
vllm/model_executor/models/prithvi_geospatial_mae.py
+4
-3
@@ -34,7 +34,8 @@ from vllm.model_executor.models.utils import AutoWeightsLoader
34
from vllm.multimodal import MULTIMODAL_REGISTRY
35
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36
MultiModalFieldElem, MultiModalInputs,
37
- MultiModalKwargs, MultiModalKwargsItem,
37
+ MultiModalKwargsItem,
38
+ MultiModalKwargsItems,
39
MultiModalSharedField, PlaceholderRange)
40
from vllm.multimodal.parse import MultiModalDataItems
41
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -88,7 +89,7 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
89
self,
90
mm_items: MultiModalDataItems,
91
hf_processor_mm_kwargs: Mapping[str, object],
91
- out_mm_kwargs: MultiModalKwargs,
92
+ out_mm_kwargs: MultiModalKwargsItems,
93
) -> Sequence[PromptUpdate]:
94
return []
95
@@ -136,7 +137,7 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
137
type="multimodal",
138
prompt=prompt,
139
prompt_token_ids=[1],
139
- mm_kwargs=MultiModalKwargs(multimodal_kwargs_items),
140
+ mm_kwargs=MultiModalKwargsItems.from_seq(multimodal_kwargs_items),
141
mm_hashes=None,
142
mm_placeholders=mm_placeholders,
143
)
vllm/model_executor/models/qwen2_5_omni_thinker.py
+8
-7
@@ -54,7 +54,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
54
from vllm.multimodal import MULTIMODAL_REGISTRY
55
from vllm.multimodal.inputs import (ImageItem, ModalityData,
56
MultiModalDataDict, MultiModalFieldConfig,
57
- MultiModalKwargs, NestedTensors)
57
+ MultiModalKwargsItems, NestedTensors)
58
from vllm.multimodal.parse import (AudioProcessorItems, DictEmbeddingItems,
59
ModalityDataItems, MultiModalDataItems,
60
MultiModalDataParser)
@@ -265,7 +265,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
265
mm_items: MultiModalDataItems,
266
hf_processor_mm_kwargs: Mapping[str, object],
267
prompt_ids: list[int],
268
- mm_kwargs: MultiModalKwargs,
268
+ mm_kwargs: MultiModalKwargsItems,
269
is_update_applied: bool,
270
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
271
"""
@@ -325,7 +325,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
325
self,
326
mm_items: MultiModalDataItems,
327
hf_processor_mm_kwargs: Mapping[str, Any],
328
- out_mm_kwargs: MultiModalKwargs,
328
+ out_mm_kwargs: MultiModalKwargsItems,
329
) -> Sequence[PromptUpdate]:
330
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
331
tokenizer = self.info.get_tokenizer()
@@ -340,8 +340,9 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
340
image_token_id = vocab[image_token]
341
video_token_id = vocab[video_token]
342
343
- audio_feature_lengths = out_mm_kwargs.get("audio_feature_lengths")
344
- feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
343
+ out_mm_data = out_mm_kwargs.get_data()
344
+ audio_feature_lengths = out_mm_data.get("audio_feature_lengths")
345
+ feature_attention_mask = out_mm_data.get("feature_attention_mask")
346
if audio_feature_lengths is None and feature_attention_mask is None:
347
audio_output_lengths = []
348
elif audio_feature_lengths is not None:
@@ -371,7 +372,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
372
return [audio_token_id] * num_features
373
374
def get_replacement_qwen2_vision(item_idx: int, modality: str):
374
- grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
375
+ grid_thw = out_mm_data[f"{modality}_grid_thw"][item_idx]
376
assert isinstance(grid_thw, torch.Tensor)
377
merge_length = image_processor.merge_size**2
378
@@ -387,7 +388,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
388
389
audio_num_features = audio_output_lengths[audio_in_video_item_idx +
390
item_idx]
390
- video_grid_thw = out_mm_kwargs["video_grid_thw"][item_idx]
391
+ video_grid_thw = out_mm_data["video_grid_thw"][item_idx]
392
393
audio_in_video_item_idx += 1
394
vllm/model_executor/models/qwen2_audio.py
+4
-3
@@ -37,7 +37,7 @@ from vllm.config import VllmConfig
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.multimodal import MULTIMODAL_REGISTRY
39
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
40
- MultiModalKwargs)
40
+ MultiModalKwargsItems)
41
from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
42
MultiModalDataParser)
43
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -182,7 +182,7 @@ class Qwen2AudioMultiModalProcessor(
182
self,
183
mm_items: MultiModalDataItems,
184
hf_processor_mm_kwargs: Mapping[str, object],
185
- out_mm_kwargs: MultiModalKwargs,
185
+ out_mm_kwargs: MultiModalKwargsItems,
186
) -> Sequence[PromptUpdate]:
187
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
188
tokenizer = self.info.get_tokenizer()
@@ -199,7 +199,8 @@ class Qwen2AudioMultiModalProcessor(
199
audio_bos_id = vocab[audio_bos_token]
200
audio_eos_id = vocab[audio_eos_token]
201
202
- feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
202
+ out_mm_data = out_mm_kwargs.get_data()
203
+ feature_attention_mask = out_mm_data.get("feature_attention_mask")
204
if feature_attention_mask is None:
205
audio_output_lengths = []
206
else:
vllm/model_executor/models/qwen2_vl.py
+4
-3
@@ -58,7 +58,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
58
from vllm.multimodal import MULTIMODAL_REGISTRY
59
from vllm.multimodal.inputs import (ImageItem, ModalityData,
60
MultiModalDataDict, MultiModalFieldConfig,
61
- MultiModalKwargs, VideoItem)
61
+ MultiModalKwargsItems, VideoItem)
62
from vllm.multimodal.parse import (DictEmbeddingItems, ImageSize,
63
ModalityDataItems, MultiModalDataItems,
64
MultiModalDataParser)
@@ -975,7 +975,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
975
self,
976
mm_items: MultiModalDataItems,
977
hf_processor_mm_kwargs: Mapping[str, Any],
978
- out_mm_kwargs: MultiModalKwargs,
978
+ out_mm_kwargs: MultiModalKwargsItems,
979
) -> Sequence[PromptUpdate]:
980
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
981
image_processor = self.info.get_image_processor(
@@ -991,7 +991,8 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
991
merge_length = image_processor.merge_size**2
992
993
def get_replacement_qwen2vl(item_idx: int, modality: str):
994
- grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
994
+ out_item = out_mm_kwargs[modality][item_idx]
995
+ grid_thw = out_item[f"{modality}_grid_thw"].data
996
assert isinstance(grid_thw, torch.Tensor)
997
998
num_tokens = int(grid_thw.prod()) // merge_length
vllm/model_executor/models/qwen_vl.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
33
from vllm.model_executor.models.module_mapping import MultiModelKeys
34
from vllm.multimodal import MULTIMODAL_REGISTRY
35
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36
- MultiModalKwargs)
36
+ MultiModalKwargsItems)
37
from vllm.multimodal.parse import MultiModalDataItems
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
BaseProcessingInfo, PromptReplacement,
@@ -627,7 +627,7 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
627
self,
628
mm_items: MultiModalDataItems,
629
hf_processor_mm_kwargs: Mapping[str, object],
630
- out_mm_kwargs: MultiModalKwargs,
630
+ out_mm_kwargs: MultiModalKwargsItems,
631
) -> Sequence[PromptUpdate]:
632
tokenizer = self.info.get_tokenizer()
633
special_tokens: dict[str,
vllm/model_executor/models/skyworkr1v.py
+7
-6
@@ -26,7 +26,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
26
from vllm.multimodal import MULTIMODAL_REGISTRY
27
from vllm.multimodal.image import convert_image_mode
28
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
29
- MultiModalKwargs, NestedTensors)
29
+ MultiModalKwargsItems, NestedTensors)
30
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
31
ImageSize, MultiModalDataItems)
32
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -552,18 +552,19 @@ class SkyworkR1VMultiModalProcessor(
552
self,
553
mm_items: MultiModalDataItems,
554
hf_processor_mm_kwargs: Mapping[str, object],
555
- out_mm_kwargs: MultiModalKwargs,
555
+ out_mm_kwargs: MultiModalKwargsItems,
556
) -> Sequence[PromptUpdate]:
557
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
558
559
- if "image_num_patches" in out_mm_kwargs:
560
- image_num_patches = out_mm_kwargs["image_num_patches"]
559
+ out_mm_data = out_mm_kwargs.get_data()
560
+ if "image_num_patches" in out_mm_data:
561
+ image_num_patches = out_mm_data["image_num_patches"]
562
assert isinstance(image_num_patches, torch.Tensor)
563
image_num_patches = image_num_patches.tolist()
563
- elif "image_embeds" in out_mm_kwargs:
564
+ elif "image_embeds" in out_mm_data:
565
# TODO: Use image size information in dictionary embedding inputs
566
# to compute num_patches (similar to Qwen2-VL)
566
- image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
567
+ image_num_patches = [None] * len(out_mm_data["image_embeds"])
568
else:
569
image_num_patches = []
570
vllm/model_executor/models/step3_vl.py
+6
-8
@@ -28,7 +28,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
from vllm.multimodal import MULTIMODAL_REGISTRY
30
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
31
- MultiModalKwargs, NestedTensors)
31
+ MultiModalKwargsItems, NestedTensors)
32
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
33
from vllm.multimodal.processing import (BaseMultiModalProcessor,
34
BaseProcessingInfo, PromptReplacement,
@@ -520,20 +520,18 @@ class Step3VLMultiModalProcessor(BaseMultiModalProcessor[Step3VLProcessingInfo]
520
self,
521
mm_items: MultiModalDataItems,
522
hf_processor_mm_kwargs: Mapping[str, Any],
523
- out_mm_kwargs: MultiModalKwargs,
523
+ out_mm_kwargs: MultiModalKwargsItems,
524
) -> Sequence[PromptUpdate]:
525
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
526
image_placeholder_token_id = hf_processor.image_token_id
527
- batch_num_patches = out_mm_kwargs["num_patches"].tolist()
527
528
def get_replacement_step1o(item_idx: int):
530
- img_out = out_mm_kwargs.get_item("image", item_idx)
531
- num_patches = batch_num_patches[item_idx]
529
+ out_item = out_mm_kwargs["image"][item_idx]
530
+ num_patches = int(out_item["num_patches"].data)
531
if num_patches > 0:
533
- patch_newline_mask = img_out["patch_newline_mask"].data.tolist(
534
- )
532
+ patch_newline_mask = out_item["patch_newline_mask"].data
533
image_repl_ids = hf_processor._get_image_repl_features(
536
- 1, num_patches, patch_newline_mask)[1]
534
+ 1, num_patches, patch_newline_mask.tolist())[1]
535
else:
536
image_repl_ids = hf_processor._get_image_repl_features(
537
1, 0, None)[1]
vllm/model_executor/models/tarsier.py
+2
-2
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
25
from vllm.model_executor.models.llava import LlavaDummyInputsBuilder
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
27
from vllm.multimodal import MULTIMODAL_REGISTRY
28
-from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
28
+from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
29
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
30
ImageSize, MultiModalDataItems)
31
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -275,7 +275,7 @@ class TarsierMultiModalProcessor(BaseMultiModalProcessor[_I_Tarsier]):
275
self,
276
mm_items: MultiModalDataItems,
277
hf_processor_mm_kwargs: Mapping[str, object],
278
- out_mm_kwargs: MultiModalKwargs,
278
+ out_mm_kwargs: MultiModalKwargsItems,
279
) -> Sequence[PromptUpdate]:
280
hf_config = self.info.get_hf_config()
281
image_token_id = hf_config.image_token_index # The <IMAGE> token ID
vllm/model_executor/models/transformers.py
+3
-3
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
ParallelLMHead, VocabParallelEmbedding)
43
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
44
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
45
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
46
MultiModalInputs, PlaceholderRange)
47
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
@@ -237,7 +237,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
237
self,
238
mm_items: MultiModalDataItems,
239
hf_processor_mm_kwargs: Mapping[str, object],
240
- out_mm_kwargs: MultiModalKwargs,
240
+ out_mm_kwargs: MultiModalKwargsItems,
241
):
242
"""
243
Given the original multi-modal items for this modality
@@ -372,7 +372,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
372
mm_tokens_per_modality["num_image_patches"]
373
) if "num_image_patches" in mm_tokens_per_modality else None
374
processed_data['num_image_patches'] = num_image_patches
375
- mm_kwargs = MultiModalKwargs.from_hf_inputs(
375
+ mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
376
processed_data,
377
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs,
378
num_image_patches),
vllm/model_executor/models/ultravox.py
+5
-4
@@ -23,7 +23,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
26
- MultiModalKwargs, NestedTensors)
26
+ MultiModalKwargsItems, NestedTensors)
27
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
28
from vllm.multimodal.processing import (BaseMultiModalProcessor,
29
BaseProcessingInfo, PromptReplacement,
@@ -194,7 +194,7 @@ class UltravoxMultiModalProcessor(
194
self,
195
mm_items: MultiModalDataItems,
196
hf_processor_mm_kwargs: Mapping[str, Any],
197
- out_mm_kwargs: MultiModalKwargs,
197
+ out_mm_kwargs: MultiModalKwargsItems,
198
) -> Sequence[PromptUpdate]:
199
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
200
@@ -203,7 +203,8 @@ class UltravoxMultiModalProcessor(
203
# Each audio can be split into multiple chunks.
204
# chunks_start_idx[i] indicates the start index of the chunks
205
# belonging to the i-th audio.
206
- num_chunks = out_mm_kwargs.get("audio_num_chunks", torch.zeros(0))
206
+ out_mm_data = out_mm_kwargs.get_data()
207
+ num_chunks = out_mm_data.get("audio_num_chunks", torch.zeros(0))
208
chunks_start_idx: torch.Tensor = torch.cumsum(num_chunks,
209
dim=0,
210
dtype=torch.int32)
@@ -213,7 +214,7 @@ class UltravoxMultiModalProcessor(
214
def get_replacement_ultravox(item_idx: int):
215
start = chunks_start_idx[item_idx]
216
end = chunks_start_idx[item_idx + 1]
216
- audio_token_len = out_mm_kwargs["audio_token_len"][start:end].sum()
217
+ audio_token_len = out_mm_data["audio_token_len"][start:end].sum()
218
return [replacement_id] * int(audio_token_len) # type: ignore
219
220
return [
vllm/model_executor/models/voxtral.py
+4
-3
@@ -31,7 +31,7 @@ from vllm.model_executor.models.whisper import WhisperEncoder
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
32
from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34
- MultiModalKwargs, NestedTensors)
34
+ MultiModalKwargsItems, NestedTensors)
35
from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
36
MultiModalDataParser)
37
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -259,7 +259,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]
259
self,
260
mm_items: MultiModalDataItems,
261
hf_processor_mm_kwargs: Mapping[str, object],
262
- out_mm_kwargs: MultiModalKwargs,
262
+ out_mm_kwargs: MultiModalKwargsItems,
263
) -> Sequence[PromptUpdate]:
264
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
265
@@ -289,7 +289,8 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]
289
tokenization_kwargs: Mapping[str, object],
290
*,
291
return_mm_hashes: bool,
292
- ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
292
+ ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
293
+ bool]:
294
prompt_ids, mm_kwargs, mm_hashes, _ = super(
295
)._cached_apply_hf_processor(
296
prompt=prompt,
vllm/model_executor/models/whisper.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
34
from vllm.multimodal import MULTIMODAL_REGISTRY, NestedTensors
35
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36
- MultiModalKwargs)
36
+ MultiModalKwargsItems)
37
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
38
from vllm.multimodal.processing import (BaseProcessingInfo,
39
EncDecMultiModalProcessor,
@@ -728,7 +728,7 @@ class WhisperMultiModalProcessor(
728
self,
729
mm_items: MultiModalDataItems,
730
hf_processor_mm_kwargs: Mapping[str, object],
731
- out_mm_kwargs: MultiModalKwargs,
731
+ out_mm_kwargs: MultiModalKwargsItems,
732
) -> Sequence[PromptUpdate]:
733
num_tokens = self.info.get_num_audio_tokens()
734
return [
vllm/multimodal/__init__.py
+3
-1
@@ -4,7 +4,8 @@ from .base import MultiModalPlaceholderMap
4
from .hasher import MultiModalHashDict, MultiModalHasher
5
from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
6
MultiModalDataDict, MultiModalKwargs,
7
- MultiModalPlaceholderDict, NestedTensors)
7
+ MultiModalKwargsItems, MultiModalPlaceholderDict,
8
+ NestedTensors)
9
from .registry import MultiModalRegistry
10
11
MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -25,6 +26,7 @@ __all__ = [
26
"MultiModalHashDict",
27
"MultiModalHasher",
28
"MultiModalKwargs",
29
+ "MultiModalKwargsItems",
30
"MultiModalPlaceholderDict",
31
"MultiModalPlaceholderMap",
32
"NestedTensors",
vllm/multimodal/base.py
+3
-6
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Generic, NamedTuple, TypeVar
9
if TYPE_CHECKING:
10
from vllm.sequence import SequenceGroupMetadata
11
12
-from .inputs import MultiModalKwargs, NestedTensors, PlaceholderRange
12
+from .inputs import MultiModalKwargs, PlaceholderRange
13
14
_T = TypeVar("_T")
15
@@ -56,8 +56,7 @@ class MultiModalPlaceholderMap:
56
@classmethod
57
def from_seq_group(
58
cls, seq_group: "SequenceGroupMetadata", positions: range
59
- ) -> tuple[dict[str, NestedTensors], dict[str,
60
- "MultiModalPlaceholderMap"]]:
59
+ ) -> tuple[MultiModalKwargs, dict[str, "MultiModalPlaceholderMap"]]:
60
"""
61
Returns the multi-modal items that intersect with the portion of a
62
prompt (``seq_group``) represented by ``positions``, as well as a
@@ -100,7 +99,7 @@ class MultiModalPlaceholderMap:
99
seq_mm_placeholders = seq_group.multi_modal_placeholders
100
101
if not seq_mm_data or not seq_mm_placeholders:
103
- return MultiModalKwargs().get_data(), {}
102
+ return MultiModalKwargs(), {}
103
104
placeholder_maps = dict[str, MultiModalPlaceholderMap]()
105
@@ -117,8 +116,6 @@ class MultiModalPlaceholderMap:
116
117
placeholder_maps[modality] = placeholder_map
118
120
- seq_mm_data = seq_mm_data if isinstance(
121
- seq_mm_data, dict) else seq_mm_data.get_data()
119
return seq_mm_data, placeholder_maps
120
121
def append_items_from_seq_group(
vllm/multimodal/cache.py
+13
-8
@@ -11,7 +11,9 @@ from vllm.logger import init_logger
11
from vllm.utils import GiB_bytes, LRUCache
12
from vllm.utils.jsontree import json_map_leaves, json_reduce_leaves
13
14
-from .inputs import MultiModalKwargs, MultiModalKwargsItem, NestedTensors
14
+from .inputs import (MultiModalFieldElem, MultiModalKwargs,
15
+ MultiModalKwargsItem, MultiModalKwargsItems,
16
+ NestedTensors)
17
18
logger = init_logger(__name__)
19
@@ -26,8 +28,9 @@ class MultiModalCacheItemMetadata:
28
29
30
MultiModalCacheValue = Union[
29
- MultiModalKwargs,
31
+ MultiModalKwargsItems,
32
MultiModalKwargsItem,
33
+ MultiModalKwargs,
34
Mapping[str, NestedTensors],
35
MultiModalCacheItemMetadata,
36
]
@@ -44,14 +47,16 @@ class MultiModalCache:
47
*,
48
debug: bool = False,
49
) -> int:
47
- # MultiModalKwargs is not a subclass of dict
48
- if isinstance(leaf, MultiModalKwargs):
49
- return cls.get_item_size(leaf.get_data(), debug=debug)
50
+ if isinstance(leaf, MultiModalFieldElem):
51
+ return cls.get_item_size(leaf.data) # type: ignore
52
51
- # MultiModalKwargsItem is not a subclass of dict
53
+ # These are not subclasses of dict
54
+ if isinstance(leaf, MultiModalKwargsItems):
55
+ return cls.get_item_size(leaf.data) # type: ignore
56
if isinstance(leaf, MultiModalKwargsItem):
53
- leaf_data = {k: v.data for k, v in leaf.items()}
54
- return cls.get_item_size(leaf_data, debug=debug)
57
+ return cls.get_item_size(leaf.data) # type: ignore
58
+ if isinstance(leaf, MultiModalKwargs):
59
+ return cls.get_item_size(leaf.data) # type: ignore
60
61
# sys.getsizeof doesn't work for tensors
62
if isinstance(leaf, torch.Tensor):
vllm/multimodal/inputs.py
+72
-100
@@ -11,7 +11,7 @@ from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
11
Union, cast, final)
12
13
import numpy as np
14
-from typing_extensions import NotRequired, TypeAlias
14
+from typing_extensions import NotRequired, TypeAlias, deprecated
15
16
from vllm.utils import LazyLoader, full_groupby, is_list_of
17
from vllm.utils.jsontree import JSONTree, json_map_leaves
@@ -656,7 +656,7 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
656
def __init__(self, data: Mapping[str, MultiModalFieldElem] = {}) -> None:
657
super().__init__(data)
658
659
- modalities = {elem.modality for elem in self.data.values()}
659
+ modalities = {elem.modality for elem in self.values()}
660
assert len(modalities) == 1, f"Found different modalities={modalities}"
661
self._modality = next(iter(modalities))
662
@@ -668,16 +668,11 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
668
return {key: elem.data for key, elem in self.items()}
669
670
671
-class MultiModalKwargs:
671
+class MultiModalKwargsItems(UserDict[str, Sequence[MultiModalKwargsItem]]):
672
"""
673
- A dictionary that represents the keyword arguments to
674
- [`torch.nn.Module.forward`][].
675
-
676
- The metadata `items` enables us to obtain the keyword arguments
677
- corresponding to each data item in
678
- [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems], via
679
- [`get_item`][vllm.multimodal.inputs.MultiModalKwargs.get_item] and
680
- [`get_items`][vllm.multimodal.inputs.MultiModalKwargs.get_items].
673
+ A dictionary of
674
+ [`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem]s
675
+ by modality.
676
"""
677
678
@staticmethod
@@ -712,19 +707,64 @@ class MultiModalKwargs:
707
elems = [v[item_idx] for v in elems_in_modality.values()]
708
items.append(MultiModalKwargsItem.from_elems(elems))
709
715
- return MultiModalKwargs(items)
716
-
717
- def __init__(self, items: Sequence[MultiModalKwargsItem] = ()) -> None:
718
- super().__init__()
710
+ return MultiModalKwargsItems.from_seq(items)
711
712
+ @staticmethod
713
+ def from_seq(items: Sequence[MultiModalKwargsItem]):
714
items_by_modality = full_groupby(items, key=lambda x: x.modality)
721
- self._items_by_modality = dict(items_by_modality)
715
+ return MultiModalKwargsItems(items_by_modality)
716
723
- self._data: Optional[dict[str, NestedTensors]] = None
717
+ def __getitem__(self, modality: str):
718
+ if modality not in self:
719
+ raise KeyError(f"Modality {modality!r} not found. "
720
+ f"Available modalities: {set(self.keys())}")
721
+
722
+ return super().__getitem__(modality)
723
+
724
+ def get_data(self, *, pin_memory: bool = False) -> "MultiModalKwargs":
725
+ elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
726
+ for items in self.values():
727
+ for item in items:
728
+ for key, elem in item.items():
729
+ elems_by_key[key].append(elem)
730
+
731
+ return MultiModalKwargs({
732
+ key:
733
+ elems[0].field.reduce_data(elems, pin_memory=pin_memory)
734
+ for key, elems in elems_by_key.items() if len(elems) > 0
735
+ })
736
725
- @property
726
- def modalities(self):
727
- return self._items_by_modality.keys()
737
+
738
+class MultiModalKwargs(UserDict[str, NestedTensors]):
739
+ """
740
+ A dictionary that represents the keyword arguments to
741
+ [`torch.nn.Module.forward`][].
742
+ """
743
+
744
+ @staticmethod
745
+ @deprecated("`MultiModalKwargs.from_hf_inputs` is deprecated and "
746
+ "will be removed in v0.13. "
747
+ "Please use `MultiModalKwargsItems.from_hf_inputs` and "
748
+ "access the tensor data using `.get_data()`.")
749
+ def from_hf_inputs(
750
+ hf_inputs: "BatchFeature",
751
+ config_by_key: Mapping[str, MultiModalFieldConfig],
752
+ ):
753
+ return MultiModalKwargsItems.from_hf_inputs(hf_inputs, config_by_key) \
754
+ .get_data()
755
+
756
+ @staticmethod
757
+ @deprecated("`MultiModalKwargs.from_items` is deprecated and "
758
+ "will be removed in v0.13. "
759
+ "Please use `MultiModalKwargsItems.from_seq` and "
760
+ "access the tensor data using `.get_data()`.")
761
+ def from_items(
762
+ items: Sequence[MultiModalKwargsItem],
763
+ *,
764
+ pin_memory: bool = False,
765
+ ):
766
+ return MultiModalKwargsItems.from_seq(items) \
767
+ .get_data(pin_memory=pin_memory)
768
769
@staticmethod
770
def _try_stack(nested_tensors: NestedTensors,
@@ -813,92 +853,24 @@ class MultiModalKwargs:
853
854
return cast(BatchedTensorInputs, json_mapped)
855
816
- def keys(self):
817
- return self.get_data().keys()
818
-
819
- def values(self):
820
- return self.get_data().values()
821
-
822
- def items(self):
823
- return self.get_data().items()
824
-
825
- def get(self, key: str, /, default=None):
826
- return self.get_data().get(key, default)
827
-
828
- def pop(self, key: str, *args, **kwargs):
829
- data = dict(self.get_data())
830
- res = data.pop(key, *args, **kwargs)
831
-
832
- for items in self._items_by_modality.values():
833
- for item in items:
834
- item.pop(key, *args, **kwargs)
835
-
836
- self._data = None
837
-
838
- return res
839
-
840
- def __iter__(self):
841
- return iter(self.get_data())
842
-
856
def __getitem__(self, key: str):
844
- return self.get_data()[key]
857
+ if key not in self:
858
+ raise KeyError(f"Keyword argument {key!r} not found. "
859
+ f"Available keys: {set(self.keys())}")
860
+
861
+ return super().__getitem__(key)
862
863
def __eq__(self, other: object) -> bool:
864
if not isinstance(other, self.__class__):
865
return False
866
850
- return self._items_by_modality == other._items_by_modality
851
-
852
- def _validate_modality(self, method_name: str, modality: str) -> None:
853
- if not self._items_by_modality:
854
- raise RuntimeError(
855
- f"`{method_name}` is not supported when "
856
- "MultiModalKwargs is not initialized with `items`")
857
-
858
- if modality not in self._items_by_modality:
859
- available_modalities = set(self._items_by_modality.keys())
860
- raise KeyError(f"Modality {modality!r} not found. "
861
- f"Available modalities: {available_modalities}")
862
-
863
- def get_item_count(self, modality: str) -> int:
864
- """Get the number of items belonging to a modality."""
865
- self._validate_modality("get_item_count", modality)
866
- return len(self._items_by_modality[modality])
867
-
868
- def get_item(self, modality: str, item_index: int) -> MultiModalKwargsItem:
869
- """
870
- Get the keyword arguments corresponding to an item identified by
871
- its modality and index.
872
- """
873
- self._validate_modality("get_item", modality)
874
- return self._items_by_modality[modality][item_index]
875
-
876
- def get_items(self, modality: str) -> Sequence[MultiModalKwargsItem]:
877
- """
878
- Get the keyword arguments corresponding to each item belonging to
879
- a modality.
880
- """
881
- self._validate_modality("get_items", modality)
882
- return self._items_by_modality[modality]
883
-
884
- def get_data(self,
885
- *,
886
- pin_memory: bool = False) -> dict[str, NestedTensors]:
887
- if self._data is not None:
888
- return self._data
867
+ for k in self:
868
+ if k not in other:
869
+ return False
870
+ if not nested_tensors_equal(self[k], other[k]):
871
+ return False
872
890
- elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
891
- for items in self._items_by_modality.values():
892
- for item in items:
893
- for key, elem in item.items():
894
- elems_by_key[key].append(elem)
895
-
896
- data = {
897
- key: elems[0].field.reduce_data(elems, pin_memory=pin_memory)
898
- for key, elems in elems_by_key.items() if len(elems) > 0
899
- }
900
- self._data = data
901
- return data
873
+ return True
874
875
876
MultiModalPlaceholderDict: TypeAlias = Mapping[str, Sequence[PlaceholderRange]]
@@ -926,7 +898,7 @@ class MultiModalInputs(TypedDict):
898
token_type_ids: NotRequired[list[int]]
899
"""The token type IDs of the prompt."""
900
929
- mm_kwargs: MultiModalKwargs
901
+ mm_kwargs: MultiModalKwargsItems
902
"""Keyword arguments to be directly passed to the model after batching."""
903
904
mm_hashes: Optional["MultiModalHashDict"]
vllm/multimodal/parse.py
+4
-7
@@ -16,7 +16,7 @@ from vllm.utils import LazyLoader, is_list_of
16
from .audio import AudioResampler
17
from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
18
ImageItem, ModalityData, MultiModalDataDict,
19
- MultiModalFieldConfig, MultiModalKwargs, VideoItem)
19
+ MultiModalFieldConfig, MultiModalKwargsItems, VideoItem)
20
21
_T = TypeVar("_T")
22
_I = TypeVar("_I")
@@ -157,19 +157,16 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
157
self.fields_config = fields_config
158
self.required_fields = required_fields
159
160
- self._kwargs = MultiModalKwargs.from_hf_inputs(
160
+ self._kwargs = MultiModalKwargsItems.from_hf_inputs(
161
BatchFeature(dict(data)),
162
fields_config,
163
)
164
165
def get_count(self) -> int:
166
- return self._kwargs.get_item_count(self.modality)
166
+ return len(self._kwargs[self.modality])
167
168
def get(self, index: int) -> Mapping[str, torch.Tensor]:
169
- return {
170
- k: v.data
171
- for k, v in self._kwargs.get_item(self.modality, index).items()
172
- }
169
+ return self._kwargs[self.modality][index].get_data()
170
171
def get_processor_data(self) -> Mapping[str, object]:
172
return {}
vllm/multimodal/processing.py
+18
-20
@@ -23,8 +23,9 @@ from vllm.utils import flatten_2d_lists, full_groupby
23
from .cache import MultiModalCache
24
from .hasher import MultiModalHasher
25
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
26
- MultiModalFieldConfig, MultiModalInputs, MultiModalKwargs,
27
- MultiModalKwargsItem, PlaceholderRange)
26
+ MultiModalFieldConfig, MultiModalInputs,
27
+ MultiModalKwargsItem, MultiModalKwargsItems,
28
+ PlaceholderRange)
29
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
30
MultiModalDataParser)
31
@@ -985,7 +986,7 @@ _I = TypeVar("_I", bound=BaseProcessingInfo)
986
MultiModalHashes = dict[str, list[str]]
987
"""
988
A collection of hashes with a similar structure as
988
-[`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs].
989
+[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
990
"""
991
992
@@ -1095,7 +1096,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1096
self,
1097
mm_items: MultiModalDataItems,
1098
hf_processor_mm_kwargs: Mapping[str, object],
1098
- out_mm_kwargs: MultiModalKwargs,
1099
+ out_mm_kwargs: MultiModalKwargsItems,
1100
) -> Sequence[PromptUpdate]:
1101
"""
1102
Given the original multi-modal items for this modality
@@ -1361,7 +1362,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1362
self,
1363
cache: ProcessingCache,
1364
mm_cache_items_or_hashes: dict[str, list[_CacheItemOrHash]],
1364
- mm_missing_kwargs: MultiModalKwargs,
1365
+ mm_missing_kwargs: MultiModalKwargsItems,
1366
) -> dict[str, list[MultiModalKwargsItem]]:
1367
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1368
@@ -1369,10 +1370,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1370
for modality, items_or_hashes in mm_cache_items_or_hashes.items():
1371
for item_or_hash in items_or_hashes:
1372
if isinstance(item_or_hash, str):
1372
- kw_item = mm_missing_kwargs.get_item(
1373
- modality,
1374
- mm_missing_next_idx[modality],
1375
- )
1373
+ kw_item = mm_missing_kwargs[modality][
1374
+ mm_missing_next_idx[modality]]
1375
cache.put(item_or_hash, kw_item)
1376
mm_missing_next_idx[modality] += 1
1377
else:
@@ -1390,7 +1389,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1389
tokenization_kwargs: Mapping[str, object],
1390
*,
1391
return_mm_hashes: bool,
1393
- ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
1392
+ ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
1393
+ bool]:
1394
(
1395
prompt_ids,
1396
mm_processed_data,
@@ -1403,7 +1403,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1403
enable_hf_prompt_update=True,
1404
)
1405
1406
- mm_kwargs = MultiModalKwargs.from_hf_inputs(
1406
+ mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1407
mm_processed_data,
1408
self._get_mm_fields_config(mm_processed_data,
1409
hf_processor_mm_kwargs),
@@ -1423,7 +1423,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1423
tokenization_kwargs: Mapping[str, object],
1424
*,
1425
return_mm_hashes: bool,
1426
- ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
1426
+ ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
1427
+ bool]:
1428
"""
1429
Apply the HF processor on the full prompt text,
1430
caching the results and reusing cached results.
@@ -1468,7 +1469,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1469
enable_hf_prompt_update=False,
1470
)
1471
1471
- mm_missing_kwargs = MultiModalKwargs.from_hf_inputs(
1472
+ mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1473
mm_missing_processed_data,
1474
self._get_mm_fields_config(mm_missing_processed_data,
1475
hf_processor_mm_kwargs),
@@ -1480,7 +1481,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1481
mm_missing_kwargs=mm_missing_kwargs,
1482
)
1483
1483
- mm_kwargs = MultiModalKwargs([
1484
+ mm_kwargs = MultiModalKwargsItems.from_seq([
1485
item for cache_items in mm_cache_items_merged.values()
1486
for item in cache_items
1487
])
@@ -1585,14 +1586,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1586
1587
def _validate_mm_kwargs(
1588
self,
1588
- mm_kwargs: MultiModalKwargs,
1589
+ mm_kwargs: MultiModalKwargsItems,
1590
mm_item_counts: Mapping[str, int],
1591
) -> None:
1592
for modality, item_count in mm_item_counts.items():
1592
- if modality in mm_kwargs.modalities:
1593
- items = mm_kwargs.get_items(modality)
1594
- else:
1595
- items = []
1593
+ items = mm_kwargs.get(modality, [])
1594
1595
if len(items) != item_count:
1596
raise RuntimeError(
@@ -1630,7 +1628,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1628
mm_items: MultiModalDataItems,
1629
hf_processor_mm_kwargs: Mapping[str, object],
1630
prompt_ids: list[int],
1633
- mm_kwargs: MultiModalKwargs,
1631
+ mm_kwargs: MultiModalKwargsItems,
1632
is_update_applied: bool,
1633
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1634
unbound_prompt_updates = self._get_prompt_updates(
vllm/multimodal/profiling.py
+2
-2
@@ -13,7 +13,7 @@ import vllm.envs as envs
13
from vllm.logger import init_logger
14
15
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
16
- MultiModalInputs, MultiModalKwargs,
16
+ MultiModalInputs, MultiModalKwargsItems,
17
MultiModalPlaceholderDict)
18
from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
19
EncDecMultiModalProcessor)
@@ -43,7 +43,7 @@ class DummyDecoderData(NamedTuple):
43
"""Dummy data used for profiling."""
44
45
prompt_token_ids: list[int]
46
- multi_modal_data: MultiModalKwargs
46
+ multi_modal_data: MultiModalKwargsItems
47
multi_modal_placeholders: MultiModalPlaceholderDict
48
49
vllm/multimodal/utils.py
+16
-9
@@ -32,11 +32,13 @@ _M = TypeVar("_M")
32
33
if TYPE_CHECKING:
34
from .inputs import (BatchedTensorInputs, MultiModalKwargs,
35
- MultiModalKwargsItem, MultiModalPlaceholderDict)
35
+ MultiModalKwargsItem, MultiModalKwargsItems,
36
+ MultiModalPlaceholderDict)
37
else:
38
BatchedTensorInputs = Any
39
MultiModalKwargs = Any
40
MultiModalKwargsItem = Any
41
+ MultiModalKwargsItems = Any
42
MultiModalPlaceholderDict = Any
43
44
global_thread_pool = ThreadPoolExecutor(
@@ -359,18 +361,20 @@ def argsort_mm_positions(
361
"`group_mm_kwargs_by_modality` and will be removed in v0.13. "
362
"Please use `group_mm_kwargs_by_modality` instead.")
363
def group_mm_inputs_by_modality(
362
- mm_inputs: list[MultiModalKwargs]) -> list[list[MultiModalKwargs]]:
364
+ mm_inputs: list[MultiModalKwargsItems]
365
+) -> list[list[MultiModalKwargsItems]]:
366
if not mm_inputs:
367
return []
368
366
- def modality_group_func(mm_input: MultiModalKwargs) -> Union[str, int]:
369
+ def modality_group_func(
370
+ mm_input: MultiModalKwargsItems) -> Union[str, int]:
371
# If the input has multiple modalities, return a id as the unique key
372
# for the mm_input input.
369
- if len(mm_input.modalities) > 1:
373
+ if len(mm_input) > 1:
374
return id(mm_input)
375
372
- elif len(mm_input.modalities) == 1:
373
- return list(mm_input.modalities)[0]
376
+ elif len(mm_input) == 1:
377
+ return next(iter(mm_input.keys()))
378
379
# FIXME(Isotr0py): Modality of mm_input from legacy pipeline is empty,
380
# this is used to make InternVL with legacy pipeline still work with v1.
@@ -397,12 +401,12 @@ def group_mm_kwargs_by_modality(
401
Yields:
402
A tuple `(modality, num_items, grouped_kwargs)`.
403
"""
400
- from vllm.multimodal.inputs import MultiModalKwargs
404
+ from vllm.multimodal.inputs import MultiModalKwargs, MultiModalKwargsItems
405
406
for modality, items in groupby(mm_kwargs, key=lambda item: item.modality):
407
items_lst = list(items)
408
405
- # mm_kwargs_group = MultiModalKwargs(items_lst) \
409
+ # mm_kwargs_group = MultiModalKwargsItems.from_items(items_lst) \
410
# .get_data(pin_memory=pin_memory)
411
412
# if device is not None:
@@ -417,7 +421,10 @@ def group_mm_kwargs_by_modality(
421
# We will also need to update each model to remove `flatten_bn`.
422
mm_kwargs_group = MultiModalKwargs.as_kwargs(
423
MultiModalKwargs.batch(
420
- [MultiModalKwargs([item]) for item in items_lst],
424
+ [
425
+ MultiModalKwargsItems.from_seq([item]).get_data()
426
+ for item in items_lst
427
+ ],
428
pin_memory=pin_memory,
429
),
430
device=device,
vllm/sequence.py
+2
-4
@@ -22,7 +22,6 @@ from vllm.pooling_params import PoolingParams
22
from vllm.sampling_params import RequestOutputKind, SamplingParams
23
24
if TYPE_CHECKING:
25
- from vllm.multimodal.inputs import NestedTensors
25
from vllm.v1.worker.kv_connector_model_runner_mixin import (
26
KVConnectorOutput)
27
@@ -523,7 +522,7 @@ class Sequence:
522
@property
523
def multi_modal_data(self) -> MultiModalKwargs:
524
if self.inputs["type"] == "multimodal":
526
- return self.inputs["mm_kwargs"]
525
+ return self.inputs["mm_kwargs"].get_data()
526
527
return MultiModalKwargs()
528
@@ -979,8 +978,7 @@ class SequenceGroupMetadata(
978
state: Optional[SequenceGroupState] = msgspec.field(
979
default_factory=lambda: SequenceGroupState())
980
token_type_ids: Optional[list[int]] = None
982
- multi_modal_data: Optional[Union[MultiModalKwargs,
983
- dict[str, "NestedTensors"]]] = None
981
+ multi_modal_data: Optional[MultiModalKwargs] = None
982
multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None
983
encoder_seq_data: Optional[SequenceData] = None
984
cross_block_table: Optional[list[int]] = None
vllm/v1/engine/processor.py
+1
-1
@@ -310,7 +310,7 @@ class Processor:
310
sorted_mm_idxs = argsort_mm_positions(decoder_mm_positions)
311
312
orig_sorted_mm_inputs = [
313
- decoder_mm_inputs.get_item(modality, idx)
313
+ decoder_mm_inputs[modality][idx]
314
for modality, idx in sorted_mm_idxs
315
]
316
sorted_mm_positions = [
vllm/v1/serial_utils.py
+33
-8
@@ -18,12 +18,15 @@ from msgspec import msgpack
18
19
from vllm import envs
20
from vllm.logger import init_logger
21
+# yapf: disable
22
from vllm.multimodal.inputs import (BaseMultiModalField,
23
MultiModalBatchedField,
24
MultiModalFieldConfig, MultiModalFieldElem,
25
MultiModalFlatField, MultiModalKwargs,
26
MultiModalKwargsItem,
27
+ MultiModalKwargsItems,
28
MultiModalSharedField, NestedTensors)
29
+# yapf: enable
30
from vllm.v1.engine import UtilityResult
31
32
logger = init_logger(__name__)
@@ -116,12 +119,11 @@ class MsgpackEncoder:
119
if isinstance(obj, MultiModalKwargsItem):
120
return self._encode_mm_item(obj)
121
122
+ if isinstance(obj, MultiModalKwargsItems):
123
+ return self._encode_mm_items(obj)
124
+
125
if isinstance(obj, MultiModalKwargs):
120
- return [
121
- self._encode_mm_item(item)
122
- for itemlist in obj._items_by_modality.values()
123
- for item in itemlist
124
- ]
126
+ return self._encode_mm_kwargs(obj)
127
128
if isinstance(obj, UtilityResult):
129
result = obj.result
@@ -183,6 +185,12 @@ class MsgpackEncoder:
185
dtype = str(obj.dtype).removeprefix("torch.")
186
return dtype, obj.shape, data
187
188
+ def _encode_mm_items(self, items: MultiModalKwargsItems) -> dict[str, Any]:
189
+ return {
190
+ modality: [self._encode_mm_item(item) for item in itemlist]
191
+ for modality, itemlist in items.items()
192
+ }
193
+
194
def _encode_mm_item(self,
195
item: MultiModalKwargsItem) -> list[dict[str, Any]]:
196
return [self._encode_mm_field_elem(elem) for elem in item.values()]
@@ -200,6 +208,12 @@ class MsgpackEncoder:
208
self._encode_mm_field(elem.field),
209
}
210
211
+ def _encode_mm_kwargs(self, kw: MultiModalKwargs) -> dict[str, Any]:
212
+ return {
213
+ modality: self._encode_nested_tensors(data)
214
+ for modality, data in kw.items()
215
+ }
216
+
217
def _encode_nested_tensors(self, nt: NestedTensors) -> Any:
218
if isinstance(nt, torch.Tensor):
219
return self._encode_tensor(nt)
@@ -260,8 +274,10 @@ class MsgpackDecoder:
274
return slice(*obj)
275
if issubclass(t, MultiModalKwargsItem):
276
return self._decode_mm_item(obj)
277
+ if issubclass(t, MultiModalKwargsItems):
278
+ return self._decode_mm_items(obj)
279
if issubclass(t, MultiModalKwargs):
264
- return MultiModalKwargs(self._decode_mm_items(obj))
280
+ return self._decode_mm_kwargs(obj)
281
if t is UtilityResult:
282
return self._decode_utility_result(obj)
283
return obj
@@ -315,8 +331,11 @@ class MsgpackDecoder:
331
# Convert back to proper shape & type
332
return arr.view(torch_dtype).view(shape)
333
318
- def _decode_mm_items(self, obj: list[Any]) -> list[MultiModalKwargsItem]:
319
- return [self._decode_mm_item(v) for v in obj]
334
+ def _decode_mm_items(self, obj: dict[str, Any]) -> MultiModalKwargsItems:
335
+ return MultiModalKwargsItems({
336
+ modality: [self._decode_mm_item(item) for item in itemlist]
337
+ for modality, itemlist in obj.items()
338
+ })
339
340
def _decode_mm_item(self, obj: list[Any]) -> MultiModalKwargsItem:
341
return MultiModalKwargsItem.from_elems(
@@ -339,6 +358,12 @@ class MsgpackDecoder:
358
obj["field"] = factory_meth(None, *field_args).field
359
return MultiModalFieldElem(**obj)
360
361
+ def _decode_mm_kwargs(self, obj: dict[str, Any]) -> MultiModalKwargs:
362
+ return MultiModalKwargs({
363
+ modality: self._decode_nested_tensors(data)
364
+ for modality, data in obj.items()
365
+ })
366
+
367
def _decode_nested_tensors(self, obj: Any) -> NestedTensors:
368
if isinstance(obj, (int, float)):
369
# Although it violates NestedTensors type, MultiModalKwargs
vllm/v1/worker/gpu_input_batch.py
+6
-4
@@ -10,8 +10,8 @@ import torch
10
from typing_extensions import deprecated
11
12
from vllm.lora.request import LoRARequest
13
-from vllm.multimodal.inputs import (MultiModalKwargs, MultiModalKwargsItem,
14
- PlaceholderRange)
13
+from vllm.multimodal.inputs import (MultiModalKwargsItem,
14
+ MultiModalKwargsItems, PlaceholderRange)
15
from vllm.pooling_params import PoolingParams
16
from vllm.sampling_params import SamplingParams, SamplingType
17
from vllm.utils import swap_dict_values
@@ -57,8 +57,10 @@ class CachedRequestState:
57
@property
58
@deprecated("`mm_inputs` is superseded by `mm_kwargs` and will be "
59
"removed in v0.13. Please use `mm_kwargs` instead.")
60
- def mm_inputs(self) -> list[MultiModalKwargs]:
61
- return [MultiModalKwargs([item]) for item in self.mm_kwargs]
60
+ def mm_inputs(self) -> list[MultiModalKwargsItems]:
61
+ return [
62
+ MultiModalKwargsItems.from_seq([item]) for item in self.mm_kwargs
63
+ ]
64
65
def get_token_id(self, idx: int) -> int:
66
if idx < self.num_prompt_tokens:
vllm/v1/worker/gpu_model_runner.py
+3
-2
@@ -2218,11 +2218,12 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2218
dummy_mm_data = dummy_decoder_data.multi_modal_data
2219
2220
# Result in the maximum GPU consumption of the model
2221
- dummy_mm_item = dummy_mm_data.get_item(modality=modality, item_index=0)
2221
+ dummy_mm_item = dummy_mm_data[modality][0]
2222
+ dummy_mm_items = [dummy_mm_item] * max_items_per_batch
2223
2224
return next(mm_kwargs_group
2225
for _, _, mm_kwargs_group in group_mm_kwargs_by_modality(
2225
- [dummy_mm_item] * max_items_per_batch,
2226
+ dummy_mm_items,
2227
device=self.device,
2228
pin_memory=self.pin_memory,
2229
))
vllm/v1/worker/tpu_model_runner.py
+3
-2
@@ -1824,11 +1824,12 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1824
dummy_mm_data = dummy_decoder_data.multi_modal_data
1825
1826
# Result in the maximum GPU consumption of the model
1827
- dummy_mm_item = dummy_mm_data.get_item(modality=modality, item_index=0)
1827
+ dummy_mm_item = dummy_mm_data[modality][0]
1828
+ dummy_mm_items = [dummy_mm_item] * max_items_per_batch
1829
1830
return next(grouped_mm_kwargs
1831
for _, _, grouped_mm_kwargs in group_mm_kwargs_by_modality(
1831
- [dummy_mm_item] * max_items_per_batch,
1832
+ dummy_mm_items,
1833
device=self.device,
1834
pin_memory=self.pin_memory,
1835
))