A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 01:00 UTC
[Chore] Deprecate `merge_by_field_config` arg (#30035)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 5, 2025 at 01:21 UTC
b286a311c2bab639ce6edc998a9a67c0affc9cbf
19 files changed
+90
-302
tests/models/multimodal/processing/test_common.py
+2
-2
@@ -20,7 +20,7 @@ from vllm.config.multimodal import (
20
)
21
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
22
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
23
-from vllm.multimodal.inputs import MultiModalInputs
23
+from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal
24
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
25
from vllm.tokenizers import (
26
MistralTokenizer,
@@ -418,4 +418,4 @@ def _assert_inputs_equal(
418
a_data.pop(key, None)
419
b_data.pop(key, None)
420
421
- assert a_data == b_data, msg
421
+ assert batched_tensors_equal(a_data, b_data), msg
tests/models/multimodal/processing/test_glm4_1v.py
+4
-3
@@ -5,6 +5,7 @@ import pytest
5
6
from vllm.assets.video import VideoAsset
7
from vllm.multimodal import MULTIMODAL_REGISTRY
8
+from vllm.multimodal.inputs import batched_tensors_equal
9
from vllm.multimodal.video import OpenCVDynamicVideoBackend, OpenCVVideoBackend
10
11
from ...utils import build_model_context
@@ -103,7 +104,7 @@ def test_video_loader_consistency(
104
dynamic_outputs = processor.apply(prompt, dynamic_mm_data, hf_processor_mm_kwargs)
105
106
assert static_outputs["prompt_token_ids"] == dynamic_outputs["prompt_token_ids"]
106
- assert (
107
- static_outputs["mm_kwargs"].get_data()
108
- == dynamic_outputs["mm_kwargs"].get_data()
107
+ assert batched_tensors_equal(
108
+ static_outputs["mm_kwargs"].get_data(),
109
+ dynamic_outputs["mm_kwargs"].get_data(),
110
)
tests/models/multimodal/processing/test_tensor_schema.py
+2
-3
@@ -130,10 +130,9 @@ def create_batched_mm_kwargs(
130
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
131
tokenization_kwargs=processor_inputs.tokenization_kwargs,
132
)["mm_kwargs"].require_data()
133
- items = [item for modality in supported_mm_limits for item in mm_kwargs[modality]]
133
+
134
return group_mm_kwargs_by_modality(
135
- items,
136
- merge_by_field_config=model_cls.merge_by_field_config,
135
+ [item for modality in supported_mm_limits for item in mm_kwargs[modality]]
136
)
137
138
tests/multimodal/test_cache.py
+3
-6
@@ -85,12 +85,6 @@ def _dummy_items(
85
(_dummy_item("a", {"a1": 100}), 100),
86
(_dummy_item("a", {"a1": 100, "a2": 110}), 210),
87
(_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}), 460), # noqa: E501
88
- (
89
- _dummy_items(
90
- {"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}
91
- ).get_data(),
92
- 460,
93
- ), # noqa: E501
88
],
89
)
90
def test_cache_item_size(item, expected_size):
@@ -107,6 +101,9 @@ def test_cache_item_size(item, expected_size):
101
cache[""] = MultiModalProcessorCacheItemMetadata(item, [prompt_update])
102
assert cache.currsize == expected_size
103
104
+ cache[""] = item.get_data()
105
+ assert cache.currsize == expected_size
106
+
107
108
def _create_vllm_config(
109
*,
tests/multimodal/test_inputs.py
deleted
-91
@@ -1,91 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import pytest
5
-import torch
6
-
7
-from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
8
-
9
-pytestmark = pytest.mark.cpu_test
10
-
11
-
12
-def assert_nested_tensors_equal(expected: NestedTensors, actual: NestedTensors):
13
- assert type(expected) == type(actual) # noqa: E721
14
- if isinstance(expected, torch.Tensor):
15
- assert torch.equal(expected, actual)
16
- else:
17
- for expected_item, actual_item in zip(expected, actual):
18
- assert_nested_tensors_equal(expected_item, actual_item)
19
-
20
-
21
-def assert_multimodal_inputs_equal(
22
- expected: MultiModalKwargs, actual: MultiModalKwargs
23
-):
24
- assert set(expected.keys()) == set(actual.keys())
25
- for key in expected:
26
- assert_nested_tensors_equal(expected[key], actual[key])
27
-
28
-
29
-def test_multimodal_input_batch_single_tensor():
30
- t = torch.rand([1, 2])
31
- result = MultiModalKwargs.batch([{"image": t}])
32
- assert_multimodal_inputs_equal(result, {"image": t.unsqueeze(0)})
33
-
34
-
35
-def test_multimodal_input_batch_multiple_tensors():
36
- a = torch.rand([1, 1, 2])
37
- b = torch.rand([1, 1, 2])
38
- c = torch.rand([1, 1, 2])
39
- result = MultiModalKwargs.batch([{"image": a}, {"image": b}, {"image": c}])
40
- assert_multimodal_inputs_equal(result, {"image": torch.stack([a, b, c])})
41
-
42
-
43
-def test_multimodal_input_batch_multiple_heterogeneous_tensors():
44
- a = torch.rand([1, 2, 2])
45
- b = torch.rand([1, 3, 2])
46
- c = torch.rand([1, 4, 2])
47
- result = MultiModalKwargs.batch([{"image": a}, {"image": b}, {"image": c}])
48
- assert_multimodal_inputs_equal(result, {"image": [a, b, c]})
49
-
50
-
51
-def test_multimodal_input_batch_nested_tensors():
52
- a = torch.rand([2, 3])
53
- b = torch.rand([2, 3])
54
- c = torch.rand([2, 3])
55
- result = MultiModalKwargs.batch([{"image": [a]}, {"image": [b]}, {"image": [c]}])
56
- assert_multimodal_inputs_equal(
57
- result, {"image": torch.stack([a.unsqueeze(0), b.unsqueeze(0), c.unsqueeze(0)])}
58
- )
59
-
60
-
61
-def test_multimodal_input_batch_heterogeneous_lists():
62
- a = torch.rand([1, 2, 3])
63
- b = torch.rand([1, 2, 3])
64
- c = torch.rand([1, 2, 3])
65
- result = MultiModalKwargs.batch([{"image": [a, b]}, {"image": [c]}])
66
- assert_multimodal_inputs_equal(
67
- result, {"image": [torch.stack([a, b]), c.unsqueeze(0)]}
68
- )
69
-
70
-
71
-def test_multimodal_input_batch_multiple_batchable_lists():
72
- a = torch.rand([1, 2, 3])
73
- b = torch.rand([1, 2, 3])
74
- c = torch.rand([1, 2, 3])
75
- d = torch.rand([1, 2, 3])
76
- result = MultiModalKwargs.batch([{"image": [a, b]}, {"image": [c, d]}])
77
- assert_multimodal_inputs_equal(
78
- result, {"image": torch.stack([torch.stack([a, b]), torch.stack([c, d])])}
79
- )
80
-
81
-
82
-def test_multimodal_input_batch_mixed_stacking_depths():
83
- a = torch.rand([1, 2, 3])
84
- b = torch.rand([1, 3, 3])
85
- c = torch.rand([1, 4, 3])
86
-
87
- result = MultiModalKwargs.batch([{"image": [a, b]}, {"image": [c]}])
88
- assert_multimodal_inputs_equal(result, {"image": [[a, b], c.unsqueeze(0)]})
89
-
90
- result = MultiModalKwargs.batch([{"image": [a]}, {"image": [b, c]}])
91
- assert_multimodal_inputs_equal(result, {"image": [a.unsqueeze(0), [b, c]]})
vllm/model_executor/models/deepseek_ocr.py
+2
-2
@@ -27,7 +27,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
27
from vllm.multimodal.inputs import (
28
MultiModalDataDict,
29
MultiModalFieldConfig,
30
- MultiModalKwargs,
30
+ MultiModalKwargsItems,
31
NestedTensors,
32
)
33
from vllm.multimodal.parse import (
@@ -305,7 +305,7 @@ class DeepseekOCRMultiModalProcessor(
305
self,
306
mm_items: MultiModalDataItems,
307
hf_processor_mm_kwargs: Mapping[str, object],
308
- out_mm_kwargs: MultiModalKwargs,
308
+ out_mm_kwargs: MultiModalKwargsItems,
309
) -> Sequence[PromptUpdate]:
310
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
311
vllm/model_executor/models/interfaces.py
+1
-1
@@ -78,7 +78,7 @@ class SupportsMultiModal(Protocol):
78
`multimodal_config.mm_encoder_tp_mode="data"`.
79
"""
80
81
- merge_by_field_config: ClassVar[bool] = False
81
+ merge_by_field_config: ClassVar[bool] = True
82
"""
83
A flag that indicates which implementation of
84
`vllm.multimodal.utils.group_mm_kwargs_by_modality` to use.
vllm/model_executor/models/lightonocr.py
+2
-2
@@ -28,7 +28,7 @@ from vllm.model_executor.models.utils import (
28
)
29
from vllm.multimodal import MULTIMODAL_REGISTRY
30
from vllm.multimodal.cache import BaseMultiModalProcessorCache
31
-from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
31
+from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
32
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
33
from vllm.multimodal.processing import (
34
BaseMultiModalProcessor,
@@ -103,7 +103,7 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
103
self,
104
mm_items: MultiModalDataItems,
105
hf_processor_mm_kwargs: Mapping[str, object],
106
- out_mm_kwargs: MultiModalKwargs,
106
+ out_mm_kwargs: MultiModalKwargsItems,
107
) -> Sequence[PromptUpdate]:
108
hf_config = self.info.get_hf_config()
109
image_token_id = hf_config.image_token_index
vllm/model_executor/models/nano_nemotron_vl.py
+6
-6
@@ -52,7 +52,6 @@ from vllm.multimodal.evs import (
52
from vllm.multimodal.inputs import (
53
MultiModalDataDict,
54
MultiModalFieldConfig,
55
- MultiModalKwargs,
55
MultiModalKwargsItems,
56
VideoItem,
57
)
@@ -849,17 +848,18 @@ class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
848
self,
849
mm_items: MultiModalDataItems,
850
hf_processor_mm_kwargs: Mapping[str, object],
852
- out_mm_kwargs: MultiModalKwargs,
851
+ out_mm_kwargs: MultiModalKwargsItems,
852
) -> Sequence[PromptUpdate]:
853
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
854
856
- if "image_num_patches" in out_mm_kwargs:
857
- image_num_patches = out_mm_kwargs["image_num_patches"]
855
+ out_mm_data = out_mm_kwargs.get_data()
856
+ if "image_num_patches" in out_mm_data:
857
+ image_num_patches = out_mm_data["image_num_patches"]
858
assert isinstance(image_num_patches, torch.Tensor)
859
image_num_patches = image_num_patches.tolist()
860
- elif "image_embeds" in out_mm_kwargs:
860
+ elif "image_embeds" in out_mm_data:
861
# to compute num_patches (similar to Qwen2-VL)
862
- image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
862
+ image_num_patches = [None] * len(out_mm_data["image_embeds"])
863
else:
864
image_num_patches = []
865
vllm/model_executor/models/opencua.py
+2
-2
@@ -23,7 +23,7 @@ from vllm.config import VllmConfig
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
from vllm.multimodal.inputs import (
25
MultiModalFieldConfig,
26
- MultiModalKwargs,
26
+ MultiModalKwargsItems,
27
)
28
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
29
from vllm.multimodal.processing import (
@@ -153,7 +153,7 @@ class OpenCUAMultiModalProcessor(BaseMultiModalProcessor[OpenCUAProcessingInfo])
153
self,
154
mm_items: MultiModalDataItems,
155
hf_processor_mm_kwargs: Mapping[str, Any],
156
- out_mm_kwargs: MultiModalKwargs,
156
+ out_mm_kwargs: MultiModalKwargsItems,
157
) -> Sequence[PromptUpdate]:
158
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
159
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
vllm/model_executor/models/paddleocr_vl.py
+2
-2
@@ -62,7 +62,7 @@ from vllm.multimodal.inputs import (
62
MultiModalDataDict,
63
MultiModalFeatureSpec,
64
MultiModalFieldConfig,
65
- MultiModalKwargs,
65
+ MultiModalKwargsItems,
66
)
67
from vllm.multimodal.parse import (
68
ImageProcessorItems,
@@ -307,7 +307,7 @@ class PaddleOCRVLMultiModalProcessor(
307
self,
308
mm_items: MultiModalDataItems,
309
hf_processor_mm_kwargs: Mapping[str, object],
310
- out_mm_kwargs: MultiModalKwargs,
310
+ out_mm_kwargs: MultiModalKwargsItems,
311
) -> Sequence[PromptUpdate]:
312
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
313
hf_config = self.info.get_hf_config()
vllm/model_executor/models/paligemma.py
+3
-5
@@ -40,7 +40,6 @@ from .siglip import SiglipVisionModel
40
from .utils import (
41
AutoWeightsLoader,
42
WeightsMapper,
43
- flatten_bn,
43
init_vllm_registered_model,
44
maybe_prefix,
45
)
@@ -252,6 +251,8 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
251
dummy_inputs=PaliGemmaDummyInputsBuilder,
252
)
253
class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
254
+ merge_by_field_config = True
255
+
256
packed_modules_mapping = {
257
"qkv_proj": [
258
"q_proj",
@@ -327,9 +328,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
328
return None
329
330
if pixel_values is not None:
330
- pixel_values = flatten_bn(pixel_values, concat=True)
331
-
331
h = w = self.config.vision_config.image_size
332
+
333
return PaliGemmaImagePixelInputs(
334
type="pixel_values",
335
data=pixel_values,
@@ -337,8 +337,6 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
337
)
338
339
if image_embeds is not None:
340
- image_embeds = flatten_bn(image_embeds, concat=True)
341
-
340
return PaliGemmaImageEmbeddingInputs(
341
type="image_embeds",
342
data=image_embeds,
vllm/model_executor/models/qwen2_5_vl.py
+2
-2
@@ -77,7 +77,7 @@ from vllm.multimodal.evs import (
77
from vllm.multimodal.inputs import (
78
MultiModalFeatureSpec,
79
MultiModalFieldConfig,
80
- MultiModalKwargs,
80
+ MultiModalKwargsItems,
81
)
82
from vllm.multimodal.parse import MultiModalDataItems
83
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
@@ -973,7 +973,7 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
973
self,
974
mm_items: MultiModalDataItems,
975
hf_processor_mm_kwargs: Mapping[str, Any],
976
- out_mm_kwargs: MultiModalKwargs,
976
+ out_mm_kwargs: MultiModalKwargsItems,
977
) -> Sequence[PromptUpdate]:
978
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
979
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
vllm/multimodal/cache.py
+1
-8
@@ -25,7 +25,6 @@ from .inputs import (
25
MultiModalBatchedField,
26
MultiModalFeatureSpec,
27
MultiModalFieldElem,
28
- MultiModalKwargs,
28
MultiModalKwargsItem,
29
MultiModalKwargsItems,
30
NestedTensors,
@@ -90,7 +89,6 @@ MultiModalCacheValue: TypeAlias = (
89
| MultiModalProcessorCacheItemMetadata
90
| MultiModalKwargsItems
91
| MultiModalKwargsItem
93
- | MultiModalKwargs
92
| Mapping[str, NestedTensors]
93
)
94
@@ -108,12 +106,7 @@ class MultiModalCache:
106
# These are not subclasses of dict
107
if isinstance(
108
leaf,
111
- (
112
- MultiModalKwargs,
113
- MultiModalKwargsItems,
114
- MultiModalKwargsItem,
115
- MultiModalFieldElem,
116
- ),
109
+ (MultiModalKwargsItems, MultiModalKwargsItem, MultiModalFieldElem),
110
):
111
return cls.get_item_size(leaf.data) # type: ignore
112
vllm/multimodal/inputs.py
+46
-96
@@ -3,7 +3,7 @@
3
4
from abc import ABC, abstractmethod
5
from collections import UserDict, defaultdict
6
-from collections.abc import Mapping, Sequence
6
+from collections.abc import Mapping, Sequence, Set
7
from dataclasses import dataclass
8
from functools import partial
9
from itertools import accumulate
@@ -201,8 +201,10 @@ Uses a list instead of a tensor if the dimensions of each element do not match.
201
202
203
def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
204
- """Equality check between
205
- [`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects."""
204
+ """
205
+ Equality check between
206
+ [`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects.
207
+ """
208
if isinstance(a, torch.Tensor):
209
return isinstance(b, torch.Tensor) and torch.equal(a, b)
210
elif isinstance(b, torch.Tensor):
@@ -224,10 +226,24 @@ def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
226
BatchedTensorInputs: TypeAlias = dict[str, NestedTensors]
227
"""
228
A dictionary containing nested tensors which have been batched via
227
-[`MultiModalKwargs.batch`][vllm.multimodal.inputs.MultiModalKwargs.batch].
229
+[`MultiModalKwargsItems.get_data`][vllm.multimodal.inputs.MultiModalKwargsItems.get_data].
230
"""
231
232
233
+def batched_tensors_equal(a: BatchedTensorInputs, b: BatchedTensorInputs) -> bool:
234
+ """
235
+ Equality check between
236
+ [`BatchedTensorInputs`][vllm.multimodal.inputs.BatchedTensorInputs] objects.
237
+ """
238
+ for k in a:
239
+ if k not in b:
240
+ return False
241
+ if not nested_tensors_equal(a[k], b[k]):
242
+ return False
243
+
244
+ return True
245
+
246
+
247
@dataclass
248
class MultiModalFeatureSpec:
249
"""
@@ -823,7 +839,14 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
839
840
return self # type: ignore[return-value]
841
826
- def get_data(self, *, pin_memory: bool = False) -> "MultiModalKwargs":
842
+ def get_data(
843
+ self,
844
+ *,
845
+ device: torch.types.Device = None,
846
+ pin_memory: bool = False,
847
+ cpu_fields: Set[str] = frozenset(),
848
+ ) -> BatchedTensorInputs:
849
+ """Construct a dictionary of keyword arguments to pass to the model."""
850
elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
851
for modality, items in self.items():
852
for i, item in enumerate(items):
@@ -835,12 +858,23 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):
858
for key, elem in item.items():
859
elems_by_key[key].append(elem)
860
838
- return MultiModalKwargs(
839
- {
840
- key: elems[0].field.reduce_data(elems, pin_memory=pin_memory)
841
- for key, elems in elems_by_key.items()
842
- }
843
- )
861
+ data = {
862
+ key: elems[0].field.reduce_data(elems, pin_memory=pin_memory)
863
+ for key, elems in elems_by_key.items()
864
+ }
865
+
866
+ if device is not None:
867
+ for k in data.keys() - cpu_fields:
868
+ data[k] = json_map_leaves(
869
+ (
870
+ lambda x: x.to(device=device, non_blocking=True)
871
+ if isinstance(x, torch.Tensor)
872
+ else x
873
+ ),
874
+ data[k],
875
+ )
876
+
877
+ return data
878
879
880
MultiModalKwargsOptionalItems: TypeAlias = (
@@ -849,6 +883,7 @@ MultiModalKwargsOptionalItems: TypeAlias = (
883
)
884
885
886
+@deprecated("`MultiModalKwargs` is deprecated and will be removed in v0.13.")
887
class MultiModalKwargs(UserDict[str, NestedTensors]):
888
"""
889
A dictionary that represents the keyword arguments to
@@ -882,91 +917,6 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
917
):
918
return MultiModalKwargsItems.from_seq(items).get_data(pin_memory=pin_memory)
919
885
- @staticmethod
886
- def _try_stack(
887
- nested_tensors: NestedTensors, pin_memory: bool = False
888
- ) -> NestedTensors:
889
- """
890
- Stack the inner dimensions that have the same shape in
891
- a nested list of tensors.
892
-
893
- Thus, a dimension represented by a list means that the inner
894
- dimensions are different for each element along that dimension.
895
- """
896
- if isinstance(nested_tensors, torch.Tensor):
897
- return nested_tensors
898
-
899
- # TODO: Remove these once all models have been migrated
900
- if isinstance(nested_tensors, np.ndarray):
901
- return torch.from_numpy(nested_tensors)
902
- if isinstance(nested_tensors, (int, float)):
903
- return torch.tensor(nested_tensors)
904
-
905
- stacked = [MultiModalKwargs._try_stack(t, pin_memory) for t in nested_tensors]
906
- if not is_list_of(stacked, torch.Tensor, check="all"):
907
- # Only tensors (not lists) can be stacked.
908
- return stacked
909
-
910
- tensors_ = cast(list[torch.Tensor], stacked)
911
- if len(tensors_) == 1:
912
- # An optimization when `tensors_` contains only one tensor:
913
- # - produce exactly same result as `torch.stack(tensors_)`
914
- # - will achieve zero-copy if the tensor is contiguous
915
- return tensors_[0].unsqueeze(0).contiguous()
916
-
917
- if any(t.shape != tensors_[0].shape for t in tensors_):
918
- # The tensors have incompatible shapes and can't be stacked.
919
- return tensors_
920
-
921
- outputs = torch.empty(
922
- len(tensors_),
923
- *tensors_[0].shape,
924
- dtype=tensors_[0].dtype,
925
- device=tensors_[0].device,
926
- pin_memory=pin_memory,
927
- )
928
- return torch.stack(tensors_, out=outputs)
929
-
930
- @staticmethod
931
- def batch(
932
- inputs_list: list["MultiModalKwargs"], pin_memory: bool = False
933
- ) -> BatchedTensorInputs:
934
- """
935
- Batch multiple inputs together into a dictionary.
936
-
937
- The resulting dictionary has the same keys as the inputs.
938
- If the corresponding value from each input is a tensor and they all
939
- share the same shape, the output value is a single batched tensor;
940
- otherwise, the output value is a list containing the original value
941
- from each input.
942
- """
943
- if len(inputs_list) == 0:
944
- return {}
945
-
946
- # We need to consider the case where each item in the batch
947
- # contains different modalities (i.e. different keys).
948
- item_lists = defaultdict[str, list[NestedTensors]](list)
949
-
950
- for inputs in inputs_list:
951
- for k, v in inputs.items():
952
- item_lists[k].append(v)
953
-
954
- return {
955
- k: MultiModalKwargs._try_stack(item_list, pin_memory)
956
- for k, item_list in item_lists.items()
957
- }
958
-
959
- @staticmethod
960
- def as_kwargs(
961
- batched_inputs: BatchedTensorInputs,
962
- *,
963
- device: torch.types.Device,
964
- ) -> BatchedTensorInputs:
965
- return json_map_leaves(
966
- lambda x: x.to(device=device, non_blocking=True),
967
- batched_inputs,
968
- )
969
-
920
def __getitem__(self, key: str):
921
if key not in self:
922
raise KeyError(
vllm/multimodal/utils.py
+12
-47
@@ -19,7 +19,6 @@ from PIL import Image, UnidentifiedImageError
19
import vllm.envs as envs
20
from vllm.connections import HTTPConnection, global_http_connection
21
from vllm.logger import init_logger
22
-from vllm.utils.jsontree import json_map_leaves
22
from vllm.utils.registry import ExtensionManager
23
24
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
@@ -427,59 +426,25 @@ def group_mm_kwargs_by_modality(
426
Yields:
427
A tuple `(modality, num_items, grouped_kwargs)`.
428
"""
430
- if merge_by_field_config is None:
431
- raise RuntimeError(
432
- "`group_mm_kwargs_by_modality` now requires "
433
- "`merge_by_field_config` arg, please update your model runner "
434
- "according to https://github.com/vllm-project/vllm/pull/25676."
435
- )
436
- if merge_by_field_config is False:
429
+ # TODO: After v0.13, remove merge_by_field_config attribute from model impls
430
+ if merge_by_field_config is not None:
431
logger.warning_once(
438
- "The legacy code for batching multi-modal kwargs is deprecated and "
439
- "will be removed in v0.12. Please update your model with "
440
- "`merge_by_field_config=True` to use the new code defined by "
441
- "`MultiModalFieldConfig`. You can refer to "
442
- "https://github.com/vllm-project/vllm/issues/26149 "
443
- "for some examples on how to do this."
432
+ "The `merge_by_field_config` argument of `group_mm_kwargs_by_modality` "
433
+ "is deprecated and will be removed in v0.13."
434
)
435
446
- from vllm.multimodal.inputs import MultiModalKwargs, MultiModalKwargsItems
436
+ from vllm.multimodal.inputs import MultiModalKwargsItems
437
438
for modality, items in groupby(mm_kwargs, key=lambda item: item.modality):
439
items_lst = list(items)
440
+ mm_kwargs_items = MultiModalKwargsItems.from_seq(items_lst)
441
+ mm_kwargs_data = mm_kwargs_items.get_data(
442
+ device=device,
443
+ pin_memory=pin_memory,
444
+ cpu_fields=multimodal_cpu_fields,
445
+ )
446
451
- if merge_by_field_config:
452
- mm_kwargs_group: BatchedTensorInputs = dict(
453
- MultiModalKwargsItems.from_seq(items_lst).get_data(
454
- pin_memory=pin_memory
455
- )
456
- )
457
-
458
- if device is not None:
459
- mm_kwargs_group = {
460
- k: json_map_leaves(
461
- lambda x: x.to(device=device, non_blocking=True)
462
- if isinstance(x, torch.Tensor)
463
- else x,
464
- v,
465
- )
466
- if k not in multimodal_cpu_fields
467
- else v
468
- for k, v in mm_kwargs_group.items()
469
- }
470
- else:
471
- mm_kwargs_group = MultiModalKwargs.as_kwargs(
472
- MultiModalKwargs.batch(
473
- [
474
- MultiModalKwargsItems.from_seq([item]).get_data()
475
- for item in items_lst
476
- ],
477
- pin_memory=pin_memory,
478
- ),
479
- device=device,
480
- )
481
-
482
- yield modality, len(items_lst), mm_kwargs_group
447
+ yield modality, len(items_lst), mm_kwargs_data
448
449
450
def fetch_audio(
vllm/v1/serial_utils.py
-19
@@ -27,7 +27,6 @@ from vllm.multimodal.inputs import (
27
MultiModalFieldConfig,
28
MultiModalFieldElem,
29
MultiModalFlatField,
30
- MultiModalKwargs,
30
MultiModalKwargsItem,
31
MultiModalKwargsItems,
32
MultiModalSharedField,
@@ -176,9 +175,6 @@ class MsgpackEncoder:
175
if isinstance(obj, MultiModalKwargsItems):
176
return self._encode_mm_items(obj)
177
179
- if isinstance(obj, MultiModalKwargs):
180
- return self._encode_mm_kwargs(obj)
181
-
178
if isinstance(obj, UtilityResult):
179
result = obj.result
180
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
@@ -259,11 +255,6 @@ class MsgpackEncoder:
255
"field": self._encode_mm_field(elem.field),
256
}
257
262
- def _encode_mm_kwargs(self, kw: MultiModalKwargs) -> dict[str, Any]:
263
- return {
264
- modality: self._encode_nested_tensors(data) for modality, data in kw.items()
265
- }
266
-
258
def _encode_nested_tensors(self, nt: NestedTensors) -> Any:
259
if isinstance(nt, torch.Tensor):
260
return self._encode_tensor(nt)
@@ -325,8 +316,6 @@ class MsgpackDecoder:
316
return self._decode_mm_item(obj)
317
if issubclass(t, MultiModalKwargsItems):
318
return self._decode_mm_items(obj)
328
- if issubclass(t, MultiModalKwargs):
329
- return self._decode_mm_kwargs(obj)
319
if t is UtilityResult:
320
return self._decode_utility_result(obj)
321
return obj
@@ -414,14 +403,6 @@ class MsgpackDecoder:
403
obj["field"] = factory_meth(None, *field_args).field
404
return MultiModalFieldElem(**obj)
405
417
- def _decode_mm_kwargs(self, obj: dict[str, Any]) -> MultiModalKwargs:
418
- return MultiModalKwargs(
419
- {
420
- modality: self._decode_nested_tensors(data)
421
- for modality, data in obj.items()
422
- }
423
- )
424
-
406
def _decode_nested_tensors(self, obj: Any) -> NestedTensors:
407
if isinstance(obj, (int, float)):
408
# Although it violates NestedTensors type, MultiModalKwargs
vllm/v1/worker/gpu_model_runner.py
-3
@@ -2106,7 +2106,6 @@ class GPUModelRunner(
2106
mm_kwargs,
2107
device=self.device,
2108
pin_memory=self.pin_memory,
2109
- merge_by_field_config=model.merge_by_field_config,
2109
multimodal_cpu_fields=model.multimodal_cpu_fields,
2110
):
2111
curr_group_outputs: list[torch.Tensor] = []
@@ -2133,7 +2132,6 @@ class GPUModelRunner(
2132
[video_mm_kwargs_item],
2133
device=self.device,
2134
pin_memory=self.pin_memory,
2136
- merge_by_field_config=model.merge_by_field_config,
2135
multimodal_cpu_fields=model.multimodal_cpu_fields,
2136
)
2137
)
@@ -3849,7 +3847,6 @@ class GPUModelRunner(
3847
dummy_mm_items,
3848
device=self.device,
3849
pin_memory=self.pin_memory,
3852
- merge_by_field_config=model.merge_by_field_config,
3850
multimodal_cpu_fields=model.multimodal_cpu_fields,
3851
)
3852
)
vllm/v1/worker/tpu_model_runner.py
-2
@@ -969,7 +969,6 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
969
mm_kwargs,
970
device=self.device,
971
pin_memory=self.pin_memory,
972
- merge_by_field_config=model.merge_by_field_config,
972
multimodal_cpu_fields=model.multimodal_cpu_fields,
973
):
974
# Run the encoder.
@@ -2058,7 +2057,6 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2057
dummy_mm_items,
2058
device=self.device,
2059
pin_memory=self.pin_memory,
2061
- merge_by_field_config=model.merge_by_field_config,
2060
multimodal_cpu_fields=model.multimodal_cpu_fields,
2061
)
2062
)