A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] support input embeddings for qwen2vl (#8856)
whyiug committed
Sep 30, 2024 at 11:16 UTC
e01ab595d897698c9a5fe9eaebd983eb3e23470a
3 files changed
+136
-71
docs/source/models/supported_models.rst
+1
-1
@@ -281,7 +281,7 @@ Multimodal Language Models
281
-
282
* - :code:`Qwen2VLForConditionalGeneration`
283
- Qwen2-VL
284
- - Image\ :sup:`+` / Video\ :sup:`+`
284
+ - Image\ :sup:`E+` / Video\ :sup:`+`
285
- :code:`Qwen/Qwen2-VL-2B-Instruct`, :code:`Qwen/Qwen2-VL-7B-Instruct`, :code:`Qwen/Qwen2-VL-72B-Instruct`, etc.
286
-
287
* - :code:`UltravoxModel`
docs/source/models/vlm.rst
+17
@@ -60,7 +60,24 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptT
60
for o in outputs:
61
generated_text = o.outputs[0].text
62
print(generated_text)
63
+
64
+ # Inference with image embeddings as input with additional parameters
65
+ # Specifically, we are conducting a trial run of Qwen2VL with the new input format, as the model utilizes additional parameters for calculating positional encoding.
66
+ image_embeds = torch.load(...) # torch.Tensor of shape (1, image_feature_size, hidden_size of LM)
67
+ image_grid_thw = torch.load(...) # torch.Tensor of shape (1, 3)
68
+ mm_data['image'] = {
69
+ "image_embeds": image_embeds,
70
+ "image_grid_thw": image_grid_thw,
71
+ }
72
+ outputs = llm.generate({
73
+ "prompt": prompt,
74
+ "multi_modal_data": mm_data,
75
+ })
76
77
+ for o in outputs:
78
+ generated_text = o.outputs[0].text
79
+ print(generated_text)
80
+
81
# Batch inference
82
image_1 = PIL.Image.open(...)
83
image_2 = PIL.Image.open(...)
vllm/model_executor/models/qwen2_vl.py
+118
-70
@@ -23,8 +23,8 @@
23
# limitations under the License.
24
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
25
from functools import lru_cache, partial
26
-from typing import (Iterable, List, Mapping, Optional, Tuple, Type, TypedDict,
27
- Union)
26
+from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
27
+ Tuple, Type, TypedDict, Union)
28
29
import torch
30
import torch.nn as nn
@@ -76,19 +76,31 @@ logger = init_logger(__name__)
76
# === Vision Inputs === #
77
78
79
-class Qwen2VLImageInputs(TypedDict):
80
- pixel_values: torch.Tensor
79
+class Qwen2VLImagePixelInputs(TypedDict):
80
+ type: Literal["pixel_values"]
81
+ data: torch.Tensor
82
"""Shape:
83
`(num_patches, num_channels * patch_size * patch_size)`
84
"""
85
86
image_grid_thw: torch.Tensor
87
"""Shape: `(num_images, 3)`
87
-
88
This should be in `(grid_t, grid_h, grid_w)` format.
89
"""
90
91
92
+class Qwen2VLImageEmbeddingInputs(TypedDict):
93
+ type: Literal["image_embeds"]
94
+ data: torch.Tensor
95
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
96
+ `hidden_size` must match the hidden size of language model backbone.
97
+ """
98
+
99
+
100
+Qwen2VLImageInputs = Union[Qwen2VLImagePixelInputs,
101
+ Qwen2VLImageEmbeddingInputs]
102
+
103
+
104
class Qwen2VLVideoInputs(TypedDict):
105
pixel_values_videos: torch.Tensor
106
"""Shape:
@@ -567,6 +579,11 @@ def mm_input_mapper_for_qwen2_vl(
579
data_type_key: str,
580
) -> MultiModalInputs:
581
"""Input mapper for Qwen2-VL."""
582
+ if data_type_key == "image" and isinstance(data, dict):
583
+ return MultiModalInputs({
584
+ "image_embeds": data.get("image_embeds"),
585
+ "image_grid_thw": data.get("image_grid_thw"),
586
+ })
587
model_config = ctx.model_config
588
image_processor = cached_get_image_processor(
589
model_config.model, trust_remote_code=model_config.trust_remote_code)
@@ -739,6 +756,48 @@ def _get_llm_num_vision_tokens(
756
return llm_num_vision_tokens
757
758
759
+def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable,
760
+ data_type_key: str, image_processor: Any,
761
+ prompt_token_ids: List[int]) -> List[int]:
762
+ """
763
+ Expand pad tokens for multi-modal inputs (e.g., images or videos).
764
+
765
+ Args:
766
+ inputs (list): The multi-modal inputs (e.g., images or videos).
767
+ token_id (int): The token ID used to represent the multi-modal input.
768
+ make_batched_fn (Callable): A function to batch the inputs.
769
+ data_type_key (str): The type of the multi-modal input.
770
+ image_processor (Any): The image processor used to process the inputs.
771
+ prompt_token_ids (List[int]): The list of token IDs in the prompt.
772
+
773
+ Returns:
774
+ List[int]: The list of token IDs for the multi-modal inputs.
775
+ """
776
+ indices = [
777
+ idx for idx, token in enumerate(prompt_token_ids) if token == token_id
778
+ ]
779
+ inputs = make_batched_fn(inputs)
780
+ assert len(indices) == len(inputs)
781
+
782
+ prompt_token_ids_with_data = []
783
+ for cnt, data in enumerate(inputs):
784
+ num_tokens = _get_llm_num_vision_tokens(
785
+ [data] if data_type_key == "image" else data,
786
+ data_type_key=data_type_key,
787
+ image_processor=image_processor,
788
+ )
789
+ if cnt == 0:
790
+ end_idx = indices[cnt]
791
+ non_data_tokens = prompt_token_ids[:end_idx]
792
+ else:
793
+ non_data_tokens = prompt_token_ids[indices[cnt - 1] +
794
+ 1:indices[cnt]]
795
+ prompt_token_ids_with_data.extend(non_data_tokens)
796
+ prompt_token_ids_with_data.extend(token_id for _ in range(num_tokens))
797
+ prompt_token_ids_with_data.extend(prompt_token_ids[indices[-1] + 1:])
798
+ return prompt_token_ids_with_data
799
+
800
+
801
def input_processor_for_qwen2_vl(ctx: InputContext,
802
llm_inputs: LLMInputs) -> LLMInputs:
803
multi_modal_data = llm_inputs.get("multi_modal_data", None)
@@ -775,62 +834,38 @@ def input_processor_for_qwen2_vl(ctx: InputContext,
834
)["input_ids"]
835
836
# Expand image pad tokens.
837
+
838
if image_inputs is not None:
779
- image_indices = [
780
- idx for idx, token in enumerate(prompt_token_ids)
781
- if token == hf_config.image_token_id
782
- ]
783
- image_inputs = make_batched_images(image_inputs)
784
- assert len(image_indices) == len(image_inputs)
785
-
786
- prompt_token_ids_with_image = []
787
- for image_cnt, image in enumerate(image_inputs):
788
- num_image_tokens = _get_llm_num_vision_tokens(
789
- [image],
790
- data_type_key="image",
791
- image_processor=image_processor,
792
- )
793
- if image_cnt == 0:
794
- non_image_tokens = prompt_token_ids[:image_indices[image_cnt]]
795
- else:
796
- non_image_tokens = prompt_token_ids[image_indices[image_cnt -
797
- 1] +
798
- 1:image_indices[image_cnt]]
799
- prompt_token_ids_with_image.extend(non_image_tokens)
800
- prompt_token_ids_with_image.extend(
801
- hf_config.image_token_id for _ in range(num_image_tokens))
802
- prompt_token_ids_with_image.extend(prompt_token_ids[image_indices[-1] +
803
- 1:])
804
- prompt_token_ids = prompt_token_ids_with_image
805
-
806
- # Expand video pad tokens.
839
+ if isinstance(image_inputs, dict):
840
+ prompt_token_ids_with_image = []
841
+ image_indices = [
842
+ idx for idx, token in enumerate(prompt_token_ids)
843
+ if token == hf_config.image_token_id
844
+ ]
845
+ image_cnt = len(image_indices)
846
+ embed_dim = image_inputs.get('image_embeds').size(0)
847
+ assert embed_dim % image_cnt == 0
848
+ num_pad_tokens = embed_dim // image_cnt
849
+ for idx, token in enumerate(prompt_token_ids):
850
+ if idx in image_indices:
851
+ prompt_token_ids_with_image.extend([token] *
852
+ num_pad_tokens)
853
+ else:
854
+ prompt_token_ids_with_image.append(token)
855
+ prompt_token_ids = prompt_token_ids_with_image
856
+ else:
857
+ prompt_token_ids = _expand_pad_tokens(image_inputs,
858
+ hf_config.image_token_id,
859
+ make_batched_images, "image",
860
+ image_processor,
861
+ prompt_token_ids)
862
+
863
if video_inputs is not None:
808
- video_indices = [
809
- idx for idx, token in enumerate(prompt_token_ids)
810
- if token == hf_config.video_token_id
811
- ]
812
- video_inputs = make_batched_videos(video_inputs)
813
- assert len(video_indices) == len(video_inputs)
814
-
815
- prompt_token_ids_with_video = []
816
- for video_cnt, video in enumerate(video_inputs):
817
- num_video_tokens = _get_llm_num_vision_tokens(
818
- video,
819
- data_type_key="video",
820
- image_processor=image_processor,
821
- )
822
- if video_cnt == 0:
823
- non_video_tokens = prompt_token_ids[:video_indices[video_cnt]]
824
- else:
825
- non_video_tokens = prompt_token_ids[video_indices[video_cnt -
826
- 1] +
827
- 1:video_indices[video_cnt]]
828
- prompt_token_ids_with_video.extend(non_video_tokens)
829
- prompt_token_ids_with_video.extend(
830
- hf_config.video_token_id for _ in range(num_video_tokens))
831
- prompt_token_ids_with_video.extend(prompt_token_ids[video_indices[-1] +
832
- 1:])
833
- prompt_token_ids = prompt_token_ids_with_video
864
+ prompt_token_ids = _expand_pad_tokens(video_inputs,
865
+ hf_config.video_token_id,
866
+ make_batched_videos, "video",
867
+ image_processor,
868
+ prompt_token_ids)
869
870
return LLMInputs(
871
prompt_token_ids=prompt_token_ids,
@@ -910,22 +945,32 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
945
def _parse_and_validate_image_input(
946
self, **kwargs: object) -> Optional[Qwen2VLImageInputs]:
947
pixel_values = kwargs.pop("pixel_values", None)
948
+ image_embeds = kwargs.pop("image_embeds", None)
949
image_grid_thw = kwargs.pop("image_grid_thw", None)
950
915
- if pixel_values is None:
951
+ if pixel_values is None and image_embeds is None:
952
return None
953
918
- pixel_values = self._validate_and_reshape_mm_tensor(
919
- pixel_values, "image pixel values")
920
- image_grid_thw = self._validate_and_reshape_mm_tensor(
921
- image_grid_thw, "image grid_thw")
954
+ if pixel_values is not None:
955
+ pixel_values = self._validate_and_reshape_mm_tensor(
956
+ pixel_values, "image pixel values")
957
+ image_grid_thw = self._validate_and_reshape_mm_tensor(
958
+ image_grid_thw, "image grid_thw")
959
923
- if not isinstance(pixel_values, (torch.Tensor, list)):
924
- raise ValueError("Incorrect type of image pixel values. "
925
- f"Got type: {type(pixel_values)}")
960
+ if not isinstance(pixel_values, (torch.Tensor, list)):
961
+ raise ValueError("Incorrect type of image pixel values. "
962
+ f"Got type: {type(pixel_values)}")
963
927
- return Qwen2VLImageInputs(pixel_values=pixel_values,
928
- image_grid_thw=image_grid_thw)
964
+ return Qwen2VLImagePixelInputs(type="pixel_values",
965
+ data=pixel_values,
966
+ image_grid_thw=image_grid_thw)
967
+
968
+ if image_embeds is not None:
969
+ if not isinstance(image_embeds, torch.Tensor):
970
+ raise ValueError("Incorrect type of image embeddings. "
971
+ f"Got type: {type(image_embeds)}")
972
+ return Qwen2VLImageEmbeddingInputs(type="image_embeds",
973
+ data=image_embeds)
974
975
def _parse_and_validate_video_input(
976
self, **kwargs: object) -> Optional[Qwen2VLVideoInputs]:
@@ -947,7 +992,10 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
992
993
def _process_image_input(self,
994
image_input: Qwen2VLImageInputs) -> torch.Tensor:
950
- pixel_values = image_input["pixel_values"].type(self.visual.dtype)
995
+ if image_input["type"] == "image_embeds":
996
+ return image_input["data"].type(self.visual.dtype)
997
+
998
+ pixel_values = image_input["data"].type(self.visual.dtype)
999
image_embeds = self.visual(pixel_values,
1000
grid_thw=image_input["image_grid_thw"])
1001
return image_embeds