A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] support input image embedding for minicpmv (#9237)
whyiug committed
Oct 10, 2024 at 23:00 UTC
04de9057ab8099291e66ad876e78693c7c2f2ce5
3 files changed
+101
-43
docs/source/models/supported_models.rst
+1
-1
@@ -378,7 +378,7 @@ Text Generation
378
- ✅︎
379
* - :code:`MiniCPMV`
380
- MiniCPM-V
381
- - Image\ :sup:`+`
381
+ - Image\ :sup:`E+`
382
- :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
383
- ✅︎
384
- ✅︎
docs/source/models/vlm.rst
+11
-4
@@ -57,12 +57,19 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptT
57
print(generated_text)
58
59
# Inference with image embeddings as input with additional parameters
60
- # Specifically, we are conducting a trial run of Qwen2VL with the new input format, as the model utilizes additional parameters for calculating positional encoding.
61
- image_embeds = torch.load(...) # torch.Tensor of shape (1, image_feature_size, hidden_size of LM)
62
- image_grid_thw = torch.load(...) # torch.Tensor of shape (1, 3)
60
+ # Specifically, we are conducting a trial run of Qwen2VL and MiniCPM-V with the new input format, which utilizes additional parameters.
61
+ mm_data = {}
62
+
63
+ image_embeds = torch.load(...) # torch.Tensor of shape (num_images, image_feature_size, hidden_size of LM)
64
+ # For Qwen2VL, image_grid_thw is needed to calculate positional encoding.
65
+ mm_data['image'] = {
66
+ "image_embeds": image_embeds,
67
+ "image_grid_thw": torch.load(...) # torch.Tensor of shape (1, 3),
68
+ }
69
+ # For MiniCPM-V, image_size_list is needed to calculate details of the sliced image.
70
mm_data['image'] = {
71
"image_embeds": image_embeds,
65
- "image_grid_thw": image_grid_thw,
72
+ "image_size_list": [image.size] # list of image sizes
73
}
74
outputs = llm.generate({
75
"prompt": prompt,
vllm/model_executor/models/minicpmv.py
+89
-38
@@ -24,8 +24,8 @@
24
import math
25
import re
26
from functools import partial
27
-from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
28
- TypedDict)
27
+from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
28
+ Tuple, TypedDict, Union)
29
30
import torch
31
import torch.types
@@ -65,10 +65,12 @@ _KEYS_TO_MODIFY_MAPPING = {
65
"llm.lm_head": "lm_head",
66
}
67
68
+RawImageType = Union[Image.Image, torch.Tensor]
69
69
-class MiniCPMVImageInput(TypedDict):
70
+
71
+class MiniCPMVRawImageInput(TypedDict):
72
"""Input mapper input with auxiliary data for computing image bounds."""
71
- image: Image.Image
73
+ image: RawImageType
74
75
# Image bounds token ids in 0-dim scaler tensor.
76
im_start_id: torch.Tensor
@@ -78,7 +80,8 @@ class MiniCPMVImageInput(TypedDict):
80
81
82
class MiniCPMVImagePixelInputs(TypedDict):
81
- pixel_values: List[torch.Tensor]
83
+ type: Literal["pixel_values"]
84
+ data: List[torch.Tensor]
85
"""
86
Shape: `(batch_size * num_images, num_channels, height, width)`
87
@@ -101,6 +104,27 @@ class MiniCPMVImagePixelInputs(TypedDict):
104
"""
105
106
107
+class MiniCPMVImageEmbeddingInputs(TypedDict):
108
+ type: Literal["image_embeds"]
109
+ data: torch.Tensor
110
+ """
111
+ Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
112
+
113
+ `hidden_size` must match the hidden size of language model backbone.
114
+ instead of a batched tensor.
115
+ """
116
+
117
+ image_bounds: torch.Tensor
118
+ """
119
+ Shape: `(batch_size * num_images, 2)`
120
+
121
+ This should be in `(start, stop)` format.
122
+ """
123
+
124
+
125
+MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
126
+ MiniCPMVImageEmbeddingInputs]
127
+
128
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
129
130
@@ -194,22 +218,22 @@ class Resampler2_5(BaseResampler):
218
219
220
def _build_image_input(ctx: InputContext,
197
- image: Image.Image) -> MiniCPMVImageInput:
221
+ image: RawImageType) -> MiniCPMVRawImageInput:
222
tokenizer = cached_get_tokenizer(
223
ctx.model_config.tokenizer,
224
trust_remote_code=ctx.model_config.trust_remote_code)
225
if hasattr(tokenizer, "slice_start_id"):
202
- return MiniCPMVImageInput(
226
+ return MiniCPMVRawImageInput(
227
image=image,
228
im_start_id=torch.tensor(tokenizer.im_start_id),
229
im_end_id=torch.tensor(tokenizer.im_end_id),
230
slice_start_id=torch.tensor(tokenizer.slice_start_id),
231
slice_end_id=torch.tensor(tokenizer.slice_end_id))
232
else:
209
- return MiniCPMVImageInput(image=image,
210
- im_start_id=torch.tensor(
211
- tokenizer.im_start_id),
212
- im_end_id=torch.tensor(tokenizer.im_end_id))
233
+ return MiniCPMVRawImageInput(
234
+ image=image,
235
+ im_start_id=torch.tensor(tokenizer.im_start_id),
236
+ im_end_id=torch.tensor(tokenizer.im_end_id))
237
238
239
def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
@@ -280,20 +304,25 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
304
305
pattern = "(<image>./</image>)"
306
images = multi_modal_data["image"]
283
- if isinstance(images, Image.Image):
284
- images = [images]
307
image_tags = re.findall(pattern, prompt)
286
-
308
if len(image_tags) == 0:
309
new_token_ids = token_ids
310
new_prompt = prompt
311
else:
312
+ if isinstance(images, dict):
313
+ image_size_list = images.get("image_size_list")
314
+ images = [images.get("image_embeds")]
315
+ else:
316
+ if isinstance(images, Image.Image):
317
+ images = [images]
318
+ image_size_list = [image.size for image in images]
319
+
320
text_chunks = prompt.split(pattern)
321
new_prompt_chunks: List[str] = []
293
- for i in range(len(images)):
322
+ for i in range(len(image_size_list)):
323
new_prompt_chunks += [
324
text_chunks[i],
296
- get_placeholder(images[i].size, i)
325
+ get_placeholder(image_size_list[i], i)
326
]
327
new_prompt_chunks.append(text_chunks[-1])
328
new_prompt = "".join(new_prompt_chunks)
@@ -323,9 +352,15 @@ def input_mapper_for_minicpmv(ctx: InputContext, data: object):
352
if not isinstance(data, list):
353
raise ValueError(
354
"Image input must be list of MiniCPMVImageInput, got (%s)", data)
326
- batch_data = image_processor \
327
- .preprocess([img["image"] for img in data], return_tensors="pt") \
328
- .data
355
+
356
+ if len(data) > 0 and isinstance(data[0]['image'], torch.Tensor):
357
+ batch_data = {
358
+ "image_embeds": data[0]['image'],
359
+ }
360
+ else:
361
+ batch_data = image_processor \
362
+ .preprocess([img["image"] for img in data], return_tensors="pt") \
363
+ .data
364
365
if len(data) > 0:
366
batch_data["im_start_id"] = data[0]["im_start_id"]
@@ -380,7 +415,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
415
def get_embedding(
416
self,
417
input_ids: torch.Tensor,
383
- image_inputs: Optional[MiniCPMVImagePixelInputs],
418
+ image_inputs: Optional[MiniCPMVImageInputs],
419
) -> Tuple[torch.Tensor, torch.Tensor]:
420
vlm_embedding: torch.Tensor = self.llm.embed_tokens(input_ids)
421
if hasattr(self.config, "scale_emb"):
@@ -389,7 +424,12 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
424
if image_inputs is None: # No image
425
vision_hidden_states = torch.tensor([], device=input_ids.device)
426
else:
392
- vision_hidden_states = self.get_vision_hidden_states(image_inputs)
427
+ if image_inputs["type"] == "image_embeds":
428
+ vision_hidden_states = (image_inputs["data"].type(
429
+ vlm_embedding.dtype).to(vlm_embedding.device))
430
+ else:
431
+ vision_hidden_states = self.get_vision_hidden_states(
432
+ image_inputs)
433
434
# See NOTE in _parse_and_validate_inputs
435
image_bounds = image_inputs["image_bounds"]
@@ -440,9 +480,23 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
480
self,
481
input_ids: torch.Tensor,
482
**kwargs: object,
443
- ) -> Optional[MiniCPMVImagePixelInputs]:
483
+ ) -> Optional[MiniCPMVImageInputs]:
484
pixel_values = kwargs.pop("pixel_values", [])
485
tgt_sizes = kwargs.pop("tgt_sizes", [])
486
+ im_start_id = kwargs.pop("im_start_id", None)
487
+ im_end_id = kwargs.pop("im_end_id", None)
488
+ slice_start_id = kwargs.pop("slice_start_id", None)
489
+ slice_end_id = kwargs.pop("slice_end_id", None)
490
+ image_embeds = kwargs.pop("image_embeds", None)
491
+
492
+ if image_embeds is not None:
493
+ return MiniCPMVImageEmbeddingInputs(
494
+ image_bounds=self._get_image_bounds(input_ids, im_start_id,
495
+ im_end_id, slice_start_id,
496
+ slice_end_id),
497
+ data=image_embeds,
498
+ type="image_embeds",
499
+ )
500
501
if not isinstance(pixel_values, (torch.Tensor, list)):
502
raise ValueError("Incorrect type of pixel values. "
@@ -477,10 +531,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
531
if len(pixel_values_flat) == 0:
532
return None
533
480
- im_start_id = kwargs.pop("im_start_id", None)
481
- im_end_id = kwargs.pop("im_end_id", None)
482
- slice_start_id = kwargs.pop("slice_start_id", None)
483
- slice_end_id = kwargs.pop("slice_end_id", None)
534
if im_start_id is None:
535
return None
536
@@ -488,8 +538,9 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
538
image_bounds=self._get_image_bounds(input_ids, im_start_id,
539
im_end_id, slice_start_id,
540
slice_end_id),
491
- pixel_values=pixel_values_flat,
541
+ data=pixel_values_flat,
542
tgt_sizes=torch.stack(tgt_sizes_flat),
543
+ type="pixel_values",
544
)
545
546
def forward(
@@ -610,8 +661,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
661
) -> torch.Tensor:
662
raise NotImplementedError
663
613
- def get_vision_hidden_states(
614
- self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
664
+ def get_vision_hidden_states(self,
665
+ data: MiniCPMVImageInputs) -> torch.Tensor:
666
raise NotImplementedError
667
668
def is_default_weight_loading(self, name: str) -> bool:
@@ -705,9 +756,9 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
756
res.append(self.resampler(vision_embedding, tgt_size))
757
return torch.vstack(res)
758
708
- def get_vision_hidden_states(
709
- self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
710
- pixel_values = data["pixel_values"]
759
+ def get_vision_hidden_states(self,
760
+ data: MiniCPMVImageInputs) -> torch.Tensor:
761
+ pixel_values = data["data"]
762
763
return self.get_vision_embedding(pixel_values)
764
@@ -793,9 +844,9 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
844
vision_embedding = self.resampler(vision_embedding, tgt_sizes)
845
return vision_embedding
846
796
- def get_vision_hidden_states(
797
- self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
798
- pixel_values = data["pixel_values"]
847
+ def get_vision_hidden_states(self,
848
+ data: MiniCPMVImageInputs) -> torch.Tensor:
849
+ pixel_values = data["data"]
850
tgt_sizes = data["tgt_sizes"]
851
852
device = self.vpm.embeddings.position_embedding.weight.device
@@ -909,9 +960,9 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
960
)
961
return vision_embedding
962
912
- def get_vision_hidden_states(
913
- self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
914
- pixel_values = data["pixel_values"]
963
+ def get_vision_hidden_states(self,
964
+ data: MiniCPMVImageInputs) -> torch.Tensor:
965
+ pixel_values = data["data"]
966
tgt_sizes = data["tgt_sizes"]
967
968
device = self.vpm.embeddings.position_embedding.weight.device