@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 04de9057ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] support input image embedding for minicpmv (#9237)

whyiug committed Oct 10, 2024 at 23:00 UTC 04de9057ab8099291e66ad876e78693c7c2f2ce5
3 files changed +101 -43
docs/source/models/supported_models.rst
+1 -1
@@ -378,7 +378,7 @@ Text Generation
378 - ✅︎
379 * - :code:`MiniCPMV`
380 - MiniCPM-V
381 - - Image\ :sup:`+`
381 + - Image\ :sup:`E+`
382 - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
383 - ✅︎
384 - ✅︎
docs/source/models/vlm.rst
+11 -4
@@ -57,12 +57,19 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptT
57 print(generated_text)
58
59 # Inference with image embeddings as input with additional parameters
60 - # Specifically, we are conducting a trial run of Qwen2VL with the new input format, as the model utilizes additional parameters for calculating positional encoding.
61 - image_embeds = torch.load(...) # torch.Tensor of shape (1, image_feature_size, hidden_size of LM)
62 - image_grid_thw = torch.load(...) # torch.Tensor of shape (1, 3)
60 + # Specifically, we are conducting a trial run of Qwen2VL and MiniCPM-V with the new input format, which utilizes additional parameters.
61 + mm_data = {}
62 +
63 + image_embeds = torch.load(...) # torch.Tensor of shape (num_images, image_feature_size, hidden_size of LM)
64 + # For Qwen2VL, image_grid_thw is needed to calculate positional encoding.
65 + mm_data['image'] = {
66 + "image_embeds": image_embeds,
67 + "image_grid_thw": torch.load(...) # torch.Tensor of shape (1, 3),
68 + }
69 + # For MiniCPM-V, image_size_list is needed to calculate details of the sliced image.
70 mm_data['image'] = {
71 "image_embeds": image_embeds,
65 - "image_grid_thw": image_grid_thw,
72 + "image_size_list": [image.size] # list of image sizes
73 }
74 outputs = llm.generate({
75 "prompt": prompt,
vllm/model_executor/models/minicpmv.py
+89 -38
@@ -24,8 +24,8 @@
24 import math
25 import re
26 from functools import partial
27 -from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
28 - TypedDict)
27 +from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
28 + Tuple, TypedDict, Union)
29
30 import torch
31 import torch.types
@@ -65,10 +65,12 @@ _KEYS_TO_MODIFY_MAPPING = {
65 "llm.lm_head": "lm_head",
66 }
67
68 +RawImageType = Union[Image.Image, torch.Tensor]
69
69 -class MiniCPMVImageInput(TypedDict):
70 +
71 +class MiniCPMVRawImageInput(TypedDict):
72 """Input mapper input with auxiliary data for computing image bounds."""
71 - image: Image.Image
73 + image: RawImageType
74
75 # Image bounds token ids in 0-dim scaler tensor.
76 im_start_id: torch.Tensor
@@ -78,7 +80,8 @@ class MiniCPMVImageInput(TypedDict):
80
81
82 class MiniCPMVImagePixelInputs(TypedDict):
81 - pixel_values: List[torch.Tensor]
83 + type: Literal["pixel_values"]
84 + data: List[torch.Tensor]
85 """
86 Shape: `(batch_size * num_images, num_channels, height, width)`
87
@@ -101,6 +104,27 @@ class MiniCPMVImagePixelInputs(TypedDict):
104 """
105
106
107 +class MiniCPMVImageEmbeddingInputs(TypedDict):
108 + type: Literal["image_embeds"]
109 + data: torch.Tensor
110 + """
111 + Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
112 +
113 + `hidden_size` must match the hidden size of language model backbone.
114 + instead of a batched tensor.
115 + """
116 +
117 + image_bounds: torch.Tensor
118 + """
119 + Shape: `(batch_size * num_images, 2)`
120 +
121 + This should be in `(start, stop)` format.
122 + """
123 +
124 +
125 +MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
126 + MiniCPMVImageEmbeddingInputs]
127 +
128 DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
129
130
@@ -194,22 +218,22 @@ class Resampler2_5(BaseResampler):
218
219
220 def _build_image_input(ctx: InputContext,
197 - image: Image.Image) -> MiniCPMVImageInput:
221 + image: RawImageType) -> MiniCPMVRawImageInput:
222 tokenizer = cached_get_tokenizer(
223 ctx.model_config.tokenizer,
224 trust_remote_code=ctx.model_config.trust_remote_code)
225 if hasattr(tokenizer, "slice_start_id"):
202 - return MiniCPMVImageInput(
226 + return MiniCPMVRawImageInput(
227 image=image,
228 im_start_id=torch.tensor(tokenizer.im_start_id),
229 im_end_id=torch.tensor(tokenizer.im_end_id),
230 slice_start_id=torch.tensor(tokenizer.slice_start_id),
231 slice_end_id=torch.tensor(tokenizer.slice_end_id))
232 else:
209 - return MiniCPMVImageInput(image=image,
210 - im_start_id=torch.tensor(
211 - tokenizer.im_start_id),
212 - im_end_id=torch.tensor(tokenizer.im_end_id))
233 + return MiniCPMVRawImageInput(
234 + image=image,
235 + im_start_id=torch.tensor(tokenizer.im_start_id),
236 + im_end_id=torch.tensor(tokenizer.im_end_id))
237
238
239 def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
@@ -280,20 +304,25 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
304
305 pattern = "(<image>./</image>)"
306 images = multi_modal_data["image"]
283 - if isinstance(images, Image.Image):
284 - images = [images]
307 image_tags = re.findall(pattern, prompt)
286 -
308 if len(image_tags) == 0:
309 new_token_ids = token_ids
310 new_prompt = prompt
311 else:
312 + if isinstance(images, dict):
313 + image_size_list = images.get("image_size_list")
314 + images = [images.get("image_embeds")]
315 + else:
316 + if isinstance(images, Image.Image):
317 + images = [images]
318 + image_size_list = [image.size for image in images]
319 +
320 text_chunks = prompt.split(pattern)
321 new_prompt_chunks: List[str] = []
293 - for i in range(len(images)):
322 + for i in range(len(image_size_list)):
323 new_prompt_chunks += [
324 text_chunks[i],
296 - get_placeholder(images[i].size, i)
325 + get_placeholder(image_size_list[i], i)
326 ]
327 new_prompt_chunks.append(text_chunks[-1])
328 new_prompt = "".join(new_prompt_chunks)
@@ -323,9 +352,15 @@ def input_mapper_for_minicpmv(ctx: InputContext, data: object):
352 if not isinstance(data, list):
353 raise ValueError(
354 "Image input must be list of MiniCPMVImageInput, got (%s)", data)
326 - batch_data = image_processor \
327 - .preprocess([img["image"] for img in data], return_tensors="pt") \
328 - .data
355 +
356 + if len(data) > 0 and isinstance(data[0]['image'], torch.Tensor):
357 + batch_data = {
358 + "image_embeds": data[0]['image'],
359 + }
360 + else:
361 + batch_data = image_processor \
362 + .preprocess([img["image"] for img in data], return_tensors="pt") \
363 + .data
364
365 if len(data) > 0:
366 batch_data["im_start_id"] = data[0]["im_start_id"]
@@ -380,7 +415,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
415 def get_embedding(
416 self,
417 input_ids: torch.Tensor,
383 - image_inputs: Optional[MiniCPMVImagePixelInputs],
418 + image_inputs: Optional[MiniCPMVImageInputs],
419 ) -> Tuple[torch.Tensor, torch.Tensor]:
420 vlm_embedding: torch.Tensor = self.llm.embed_tokens(input_ids)
421 if hasattr(self.config, "scale_emb"):
@@ -389,7 +424,12 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
424 if image_inputs is None: # No image
425 vision_hidden_states = torch.tensor([], device=input_ids.device)
426 else:
392 - vision_hidden_states = self.get_vision_hidden_states(image_inputs)
427 + if image_inputs["type"] == "image_embeds":
428 + vision_hidden_states = (image_inputs["data"].type(
429 + vlm_embedding.dtype).to(vlm_embedding.device))
430 + else:
431 + vision_hidden_states = self.get_vision_hidden_states(
432 + image_inputs)
433
434 # See NOTE in _parse_and_validate_inputs
435 image_bounds = image_inputs["image_bounds"]
@@ -440,9 +480,23 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
480 self,
481 input_ids: torch.Tensor,
482 **kwargs: object,
443 - ) -> Optional[MiniCPMVImagePixelInputs]:
483 + ) -> Optional[MiniCPMVImageInputs]:
484 pixel_values = kwargs.pop("pixel_values", [])
485 tgt_sizes = kwargs.pop("tgt_sizes", [])
486 + im_start_id = kwargs.pop("im_start_id", None)
487 + im_end_id = kwargs.pop("im_end_id", None)
488 + slice_start_id = kwargs.pop("slice_start_id", None)
489 + slice_end_id = kwargs.pop("slice_end_id", None)
490 + image_embeds = kwargs.pop("image_embeds", None)
491 +
492 + if image_embeds is not None:
493 + return MiniCPMVImageEmbeddingInputs(
494 + image_bounds=self._get_image_bounds(input_ids, im_start_id,
495 + im_end_id, slice_start_id,
496 + slice_end_id),
497 + data=image_embeds,
498 + type="image_embeds",
499 + )
500
501 if not isinstance(pixel_values, (torch.Tensor, list)):
502 raise ValueError("Incorrect type of pixel values. "
@@ -477,10 +531,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
531 if len(pixel_values_flat) == 0:
532 return None
533
480 - im_start_id = kwargs.pop("im_start_id", None)
481 - im_end_id = kwargs.pop("im_end_id", None)
482 - slice_start_id = kwargs.pop("slice_start_id", None)
483 - slice_end_id = kwargs.pop("slice_end_id", None)
534 if im_start_id is None:
535 return None
536
@@ -488,8 +538,9 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
538 image_bounds=self._get_image_bounds(input_ids, im_start_id,
539 im_end_id, slice_start_id,
540 slice_end_id),
491 - pixel_values=pixel_values_flat,
541 + data=pixel_values_flat,
542 tgt_sizes=torch.stack(tgt_sizes_flat),
543 + type="pixel_values",
544 )
545
546 def forward(
@@ -610,8 +661,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
661 ) -> torch.Tensor:
662 raise NotImplementedError
663
613 - def get_vision_hidden_states(
614 - self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
664 + def get_vision_hidden_states(self,
665 + data: MiniCPMVImageInputs) -> torch.Tensor:
666 raise NotImplementedError
667
668 def is_default_weight_loading(self, name: str) -> bool:
@@ -705,9 +756,9 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
756 res.append(self.resampler(vision_embedding, tgt_size))
757 return torch.vstack(res)
758
708 - def get_vision_hidden_states(
709 - self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
710 - pixel_values = data["pixel_values"]
759 + def get_vision_hidden_states(self,
760 + data: MiniCPMVImageInputs) -> torch.Tensor:
761 + pixel_values = data["data"]
762
763 return self.get_vision_embedding(pixel_values)
764
@@ -793,9 +844,9 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
844 vision_embedding = self.resampler(vision_embedding, tgt_sizes)
845 return vision_embedding
846
796 - def get_vision_hidden_states(
797 - self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
798 - pixel_values = data["pixel_values"]
847 + def get_vision_hidden_states(self,
848 + data: MiniCPMVImageInputs) -> torch.Tensor:
849 + pixel_values = data["data"]
850 tgt_sizes = data["tgt_sizes"]
851
852 device = self.vpm.embeddings.position_embedding.weight.device
@@ -909,9 +960,9 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
960 )
961 return vision_embedding
962
912 - def get_vision_hidden_states(
913 - self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
914 - pixel_values = data["pixel_values"]
963 + def get_vision_hidden_states(self,
964 + data: MiniCPMVImageInputs) -> torch.Tensor:
965 + pixel_values = data["data"]
966 tgt_sizes = data["tgt_sizes"]
967
968 device = self.vpm.embeddings.position_embedding.weight.device