A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[VLM] Check required fields before initializing field config in `DictEmbeddingItems` (#13380)
Cyrus Leung committed
Feb 17, 2025 at 17:36 UTC
7b623fca0bfc561beaa3391c298656eb5187ce7a
5 files changed
+35
-22
docs/source/serving/multimodal_inputs.md
+2
-2
@@ -184,8 +184,8 @@ llm = LLM("openbmb/MiniCPM-V-2_6", trust_remote_code=True, limit_mm_per_prompt={
184
mm_data = {
185
"image": {
186
"image_embeds": image_embeds,
187
- # image_size_list is needed to calculate details of the sliced image.
188
- "image_size_list": [image.size for image in images], # list of image sizes
187
+ # image_sizes is needed to calculate details of the sliced image.
188
+ "image_sizes": [image.size for image in images], # list of image sizes
189
}
190
}
191
vllm/model_executor/models/minicpmo.py
+8
-5
@@ -23,8 +23,8 @@
23
# limitations under the License.
24
"""Inference-only MiniCPM-O model compatible with HuggingFace weights."""
25
from functools import partial
26
-from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
27
- Tuple, TypedDict, Union)
26
+from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
27
+ Optional, Set, Tuple, TypedDict, Union)
28
29
import torch
30
from torch import nn
@@ -122,13 +122,16 @@ class MiniCPMOAudioEmbeddingItems(DictEmbeddingItems):
122
def __init__(
123
self,
124
data: Mapping[str, torch.Tensor],
125
- fields_config: Mapping[str, MultiModalFieldConfig],
125
+ fields_factory: Callable[
126
+ [Mapping[str, torch.Tensor]],
127
+ Mapping[str, MultiModalFieldConfig],
128
+ ],
129
) -> None:
130
super().__init__(
131
data,
132
modality="image",
130
- fields_config=fields_config,
133
required_fields={"audio_embeds"},
134
+ fields_factory=fields_factory,
135
)
136
137
@@ -141,7 +144,7 @@ class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):
144
if isinstance(data, dict):
145
return MiniCPMOAudioEmbeddingItems(
146
data,
144
- fields_config=_minicpmo_field_config(data),
147
+ fields_factory=_minicpmo_field_config,
148
)
149
150
return super()._parse_audio_data(data)
vllm/model_executor/models/minicpmv.py
+12
-6
@@ -255,13 +255,16 @@ class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
255
def __init__(
256
self,
257
data: Mapping[str, torch.Tensor],
258
- fields_config: Mapping[str, MultiModalFieldConfig],
258
+ fields_factory: Callable[
259
+ [Mapping[str, torch.Tensor]],
260
+ Mapping[str, MultiModalFieldConfig],
261
+ ],
262
) -> None:
263
super().__init__(
264
data,
265
modality="image",
263
- fields_config=fields_config,
266
required_fields={"image_embeds", "image_sizes"},
267
+ fields_factory=fields_factory,
268
)
269
270
def get_image_size(self, index: int) -> ImageSize:
@@ -274,13 +277,16 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
277
def __init__(
278
self,
279
data: Mapping[str, torch.Tensor],
277
- fields_config: Mapping[str, MultiModalFieldConfig],
280
+ fields_factory: Callable[
281
+ [Mapping[str, torch.Tensor]],
282
+ Mapping[str, MultiModalFieldConfig],
283
+ ],
284
) -> None:
285
super().__init__(
286
data,
287
modality="video",
282
- fields_config=fields_config,
288
required_fields={"video_embeds", "video_image_sizes"},
289
+ fields_factory=fields_factory,
290
)
291
292
def get_frame_size(self, index: int) -> ImageSize:
@@ -300,7 +306,7 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
306
if isinstance(data, dict):
307
return MiniCPMVImageEmbeddingItems(
308
data,
303
- fields_config=_minicpmv_field_config(data),
309
+ fields_factory=_minicpmv_field_config,
310
)
311
312
return super()._parse_image_data(data)
@@ -312,7 +318,7 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
318
if isinstance(data, dict):
319
return MiniCPMVVideoEmbeddingItems(
320
data,
315
- fields_config=_minicpmv_field_config(data),
321
+ fields_factory=_minicpmv_field_config,
322
)
323
324
return super()._parse_video_data(data)
vllm/model_executor/models/qwen2_vl.py
+2
-2
@@ -691,8 +691,8 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
691
return DictEmbeddingItems(
692
data,
693
modality="image",
694
- fields_config=_qwen2vl_field_config(data),
694
required_fields={"image_embeds", "image_grid_thw"},
695
+ fields_factory=_qwen2vl_field_config,
696
)
697
698
return super()._parse_image_data(data)
@@ -705,8 +705,8 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
705
return DictEmbeddingItems(
706
data,
707
modality="video",
708
- fields_config=_qwen2vl_field_config(data),
708
required_fields={"video_embeds", "video_grid_thw"},
709
+ fields_factory=_qwen2vl_field_config,
710
)
711
712
return super()._parse_video_data(data)
vllm/multimodal/parse.py
+11
-7
@@ -125,17 +125,14 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
125
self,
126
data: Mapping[str, torch.Tensor],
127
modality: str,
128
- fields_config: Mapping[str, MultiModalFieldConfig],
128
required_fields: set[str],
129
+ fields_factory: Callable[
130
+ [Mapping[str, torch.Tensor]],
131
+ Mapping[str, MultiModalFieldConfig],
132
+ ],
133
) -> None:
134
super().__init__(data, modality)
135
133
- missing_required_fields = required_fields - fields_config.keys()
134
- if missing_required_fields:
135
- fields = set(fields_config.keys())
136
- msg = f"{required_fields=} should be a subset of {fields=}"
137
- raise ValueError(msg)
138
-
136
missing_required_data_keys = required_fields - data.keys()
137
if missing_required_data_keys:
138
data_keys = set(data.keys())
@@ -143,6 +140,13 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
140
f"but only found the following keys: {data_keys}")
141
raise ValueError(msg)
142
143
+ fields_config = fields_factory(data)
144
+ missing_required_fields = required_fields - fields_config.keys()
145
+ if missing_required_fields:
146
+ fields = set(fields_config.keys())
147
+ msg = f"{required_fields=} should be a subset of {fields=}"
148
+ raise ValueError(msg)
149
+
150
self.fields_config = fields_config
151
self.required_fields = required_fields
152