A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model] Initial support for BLIP-2 (#5920)
Co-authored-by: ywang96 <ywang@roblox.com>
Cyrus Leung committed
Jul 27, 2024 at 19:53 UTC
1ad86acf1789650e2ff27586e36a8159d52755dd
12 files changed
+1107
-21
docs/source/models/supported_models.rst
+8
@@ -7,6 +7,8 @@ vLLM supports a variety of generative Transformer models in `HuggingFace Transfo
7
The following is the list of model architectures that are currently supported by vLLM.
8
Alongside each architecture, we include some popular models that use it.
9
10
+----
11
+
12
Decoder-only Language Models
13
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
14
.. list-table::
@@ -186,6 +188,10 @@ Vision Language Models
188
- Models
189
- Example HuggingFace Models
190
- :ref:`LoRA <lora>`
191
+ * - :code:`Blip2ForConditionalGeneration`
192
+ - BLIP-2
193
+ - :code:`Salesforce/blip2-opt-2.7b`, :code:`Salesforce/blip2-opt-6.7b`, etc.
194
+ -
195
* - :code:`ChameleonForConditionalGeneration`
196
- Chameleon
197
- :code:`facebook/chameleon-7b` etc.
@@ -215,6 +221,8 @@ Vision Language Models
221
- :code:`openbmb/MiniCPM-V-2`, :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
222
-
223
224
+----
225
+
226
If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
227
Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` and :ref:`Enabling Multimodal Inputs <enabling_multimodal_inputs>`
228
for instructions on how to implement support for your model.
examples/offline_inference_vision_language.py
+11
@@ -106,6 +106,16 @@ def run_minicpmv(question):
106
return llm, prompt
107
108
109
+# BLIP-2
110
+def run_blip2(question):
111
+
112
+ # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
113
+ # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
114
+ prompt = f"Question: {question} Answer:"
115
+ llm = LLM(model="Salesforce/blip2-opt-2.7b")
116
+ return llm, prompt
117
+
118
+
119
model_example_map = {
120
"llava": run_llava,
121
"llava-next": run_llava_next,
@@ -114,6 +124,7 @@ model_example_map = {
124
"paligemma": run_paligemma,
125
"chameleon": run_chameleon,
126
"minicpmv": run_minicpmv,
127
+ "blip-2": run_blip2,
128
}
129
130
examples/template_blip2.jinja
new
+11
@@ -0,0 +1,11 @@
1
+{%- for message in messages -%}
2
+ {%- if message['role'] == 'user' -%}
3
+ {{- 'Question: ' + message['content'] + ' ' -}}
4
+ {%- elif message['role'] == 'assistant' -%}
5
+ {{- 'Answer: ' + message['content'] + ' ' -}}
6
+ {%- endif -%}
7
+{%- endfor -%}
8
+
9
+{%- if add_generation_prompt -%}
10
+ {{- 'Answer:' -}}
11
+{% endif %}
tests/models/test_blip2.py
new
+102
@@ -0,0 +1,102 @@
1
+from typing import List, Optional, Tuple
2
+
3
+import pytest
4
+from transformers import AutoTokenizer
5
+
6
+from vllm.multimodal.utils import rescale_image_size
7
+from vllm.sequence import SampleLogprobs
8
+
9
+from ..conftest import IMAGE_ASSETS
10
+from .utils import check_logprobs_close
11
+
12
+pytestmark = pytest.mark.vlm
13
+
14
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
15
+ "stop_sign":
16
+ "Question: What's the content of the image? Answer:",
17
+ "cherry_blossom":
18
+ "Question: What is the season? Answer:",
19
+})
20
+
21
+
22
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
23
+ Optional[SampleLogprobs]],
24
+ model: str):
25
+ """Sanitize vllm output to be comparable with hf output."""
26
+ _, output_str, out_logprobs = vllm_output
27
+
28
+ hf_output_str = output_str + "\n"
29
+
30
+ tokenizer = AutoTokenizer.from_pretrained(model)
31
+ hf_output_ids = tokenizer.encode(hf_output_str)
32
+ assert hf_output_ids[0] == tokenizer.bos_token_id
33
+ hf_output_ids = hf_output_ids[1:]
34
+
35
+ return hf_output_ids, hf_output_str, out_logprobs
36
+
37
+
38
+@pytest.mark.parametrize("model", ["Salesforce/blip2-opt-2.7b"])
39
+@pytest.mark.parametrize(
40
+ "size_factors",
41
+ [
42
+ # No image
43
+ [],
44
+ # Single-scale
45
+ [1.0],
46
+ # Single-scale, batched
47
+ [1.0, 1.0, 1.0],
48
+ # Multi-scale
49
+ [0.25, 0.5, 1.0],
50
+ ],
51
+)
52
+@pytest.mark.parametrize("dtype", ["half"])
53
+@pytest.mark.parametrize("max_tokens", [128])
54
+@pytest.mark.parametrize("num_logprobs", [5])
55
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
56
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
57
+ """Inference result should be the same between hf and vllm.
58
+
59
+ All the image fixtures for the test is under tests/images.
60
+ For huggingface runner, we provide the PIL images as input.
61
+ For vllm runner, we provide MultiModalData objects and corresponding
62
+ vision language config as input.
63
+ Note, the text input is also adjusted to abide by vllm contract.
64
+ The text output is sanitized to be able to compare with hf.
65
+ """
66
+ images = [asset.pil_image for asset in image_assets]
67
+
68
+ inputs_per_image = [(
69
+ [prompt for _ in size_factors],
70
+ [rescale_image_size(image, factor) for factor in size_factors],
71
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
72
+
73
+ # max_model_len should be greater than image_feature_size
74
+ with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
75
+ vllm_outputs_per_image = [
76
+ vllm_model.generate_greedy_logprobs(prompts,
77
+ max_tokens,
78
+ num_logprobs=num_logprobs,
79
+ images=images)
80
+ for prompts, images in inputs_per_image
81
+ ]
82
+
83
+ with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
84
+ hf_outputs_per_image = [
85
+ hf_model.generate_greedy_logprobs_limit(prompts,
86
+ max_tokens,
87
+ num_logprobs=num_logprobs,
88
+ images=images)
89
+ for prompts, images in inputs_per_image
90
+ ]
91
+
92
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
93
+ vllm_outputs_per_image):
94
+ check_logprobs_close(
95
+ outputs_0_lst=hf_outputs,
96
+ outputs_1_lst=[
97
+ vllm_to_hf_output(vllm_output, model)
98
+ for vllm_output in vllm_outputs
99
+ ],
100
+ name_0="hf",
101
+ name_1="vllm",
102
+ )
tests/models/test_fuyu.py
+4
-4
@@ -77,8 +77,8 @@ def run_test(
77
vllm_model.generate_greedy_logprobs(prompts,
78
max_tokens,
79
num_logprobs=num_logprobs,
80
- images=vllm_images)
81
- for prompts, vllm_images in inputs_per_image
80
+ images=images)
81
+ for prompts, images in inputs_per_image
82
]
83
84
with hf_runner(model, dtype=dtype) as hf_model:
@@ -89,9 +89,9 @@ def run_test(
89
hf_model.generate_greedy_logprobs_limit(prompts,
90
max_tokens,
91
num_logprobs=num_logprobs,
92
- images=hf_images,
92
+ images=images,
93
eos_token_id=eos_token_id)
94
- for prompts, hf_images in inputs_per_image
94
+ for prompts, images in inputs_per_image
95
]
96
97
for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
tests/models/test_minicpmv.py
+4
-4
@@ -88,9 +88,9 @@ def run_test(
88
vllm_model.generate_greedy_logprobs(prompts,
89
max_tokens,
90
num_logprobs=num_logprobs,
91
- images=vllm_images,
91
+ images=images,
92
stop_token_ids=stop_token_ids)
93
- for prompts, vllm_images in inputs_per_image
93
+ for prompts, images in inputs_per_image
94
]
95
96
with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
@@ -114,9 +114,9 @@ def run_test(
114
hf_model.generate_greedy_logprobs_limit(prompts,
115
max_tokens,
116
num_logprobs=num_logprobs,
117
- images=hf_images,
117
+ images=images,
118
tokenizer=tokenizer)
119
- for prompts, hf_images in inputs_per_image
119
+ for prompts, images in inputs_per_image
120
]
121
122
for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
tests/models/test_phi3v.py
+4
-4
@@ -101,8 +101,8 @@ def run_test(
101
vllm_model.generate_greedy_logprobs(prompts,
102
max_tokens,
103
num_logprobs=num_logprobs,
104
- images=vllm_images)
105
- for prompts, vllm_images in inputs_per_image
104
+ images=images)
105
+ for prompts, images in inputs_per_image
106
]
107
108
# use eager mode for hf runner, since phi3_v didn't work with flash_attn
@@ -114,9 +114,9 @@ def run_test(
114
hf_model.generate_greedy_logprobs_limit(prompts,
115
max_tokens,
116
num_logprobs=num_logprobs,
117
- images=hf_images,
117
+ images=images,
118
eos_token_id=eos_token_id)
119
- for prompts, hf_images in inputs_per_image
119
+ for prompts, images in inputs_per_image
120
]
121
122
for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
vllm/model_executor/models/__init__.py
+4
-2
@@ -16,6 +16,8 @@ _GENERATION_MODELS = {
16
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
17
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
18
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
19
+ "Blip2ForConditionalGeneration":
20
+ ("blip2", "Blip2ForConditionalGeneration"),
21
"ChameleonForConditionalGeneration":
22
("chameleon", "ChameleonForConditionalGeneration"),
23
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
@@ -56,8 +58,8 @@ _GENERATION_MODELS = {
58
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
59
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
60
"PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
59
- "PaliGemmaForConditionalGeneration":
60
- ("paligemma", "PaliGemmaForConditionalGeneration"),
61
+ "PaliGemmaForConditionalGeneration": ("paligemma",
62
+ "PaliGemmaForConditionalGeneration"),
63
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
64
"Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
65
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
vllm/model_executor/models/blip.py
new
+269
@@ -0,0 +1,269 @@
1
+"""Minimal implementation of BlipVisionModel intended to be only used
2
+within a vision language model."""
3
+from typing import Optional, Union
4
+
5
+import torch
6
+import torch.nn as nn
7
+from PIL import Image
8
+from transformers import Blip2VisionConfig, BlipVisionConfig
9
+from transformers.models.blip.modeling_blip import BlipAttention
10
+
11
+from vllm.config import ModelConfig
12
+from vllm.inputs import LLMInputs
13
+from vllm.model_executor.layers.activation import get_act_fn
14
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
15
+ RowParallelLinear)
16
+from vllm.model_executor.layers.quantization import QuantizationConfig
17
+from vllm.multimodal.image import (cached_get_tokenizer,
18
+ repeat_and_pad_image_tokens)
19
+from vllm.sequence import SequenceData
20
+
21
+
22
+def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
23
+ assert image_size % patch_size == 0
24
+ return image_size // patch_size
25
+
26
+
27
+def get_blip_num_patches(*, image_size: int, patch_size: int) -> int:
28
+ grid_length = get_blip_patch_grid_length(image_size=image_size,
29
+ patch_size=patch_size)
30
+ return grid_length * grid_length
31
+
32
+
33
+def get_blip_image_feature_size(
34
+ hf_config: Union[BlipVisionConfig, Blip2VisionConfig], ) -> int:
35
+ return get_blip_num_patches(image_size=hf_config.image_size,
36
+ patch_size=hf_config.patch_size)
37
+
38
+
39
+def get_max_blip_image_tokens(
40
+ hf_config: Union[BlipVisionConfig, Blip2VisionConfig], ) -> int:
41
+ return get_blip_image_feature_size(hf_config)
42
+
43
+
44
+def dummy_seq_data_for_blip(
45
+ hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
46
+ seq_len: int,
47
+ *,
48
+ image_token_id: int,
49
+ image_feature_size_override: Optional[int] = None,
50
+):
51
+ if image_feature_size_override is None:
52
+ image_feature_size = get_blip_image_feature_size(hf_config)
53
+ else:
54
+ image_feature_size = image_feature_size_override
55
+
56
+ token_ids = [image_token_id] * image_feature_size
57
+ token_ids += [0] * (seq_len - image_feature_size)
58
+ return SequenceData(token_ids)
59
+
60
+
61
+def dummy_image_for_blip(
62
+ hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
63
+ *,
64
+ image_width_override: Optional[int] = None,
65
+ image_height_override: Optional[int] = None,
66
+):
67
+ width = height = hf_config.image_size
68
+ if image_width_override is not None:
69
+ width = image_width_override
70
+ if image_height_override is not None:
71
+ height = image_height_override
72
+
73
+ image = Image.new("RGB", (width, height), color=0)
74
+ return {"image": image}
75
+
76
+
77
+def input_processor_for_blip(
78
+ model_config: ModelConfig,
79
+ hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
80
+ llm_inputs: LLMInputs,
81
+ *,
82
+ image_token_id: int,
83
+ image_feature_size_override: Optional[int] = None,
84
+):
85
+ multi_modal_data = llm_inputs.get("multi_modal_data")
86
+ if multi_modal_data is None or "image" not in multi_modal_data:
87
+ return llm_inputs
88
+
89
+ tokenizer = cached_get_tokenizer(model_config.tokenizer)
90
+
91
+ if image_feature_size_override is None:
92
+ image_feature_size = get_blip_image_feature_size(hf_config)
93
+ else:
94
+ image_feature_size = image_feature_size_override
95
+
96
+ new_prompt, new_token_ids = repeat_and_pad_image_tokens(
97
+ tokenizer,
98
+ llm_inputs.get("prompt"),
99
+ llm_inputs["prompt_token_ids"],
100
+ image_token_id=image_token_id,
101
+ repeat_count=image_feature_size,
102
+ )
103
+
104
+ # NOTE: Create a defensive copy of the original inputs
105
+ return LLMInputs(prompt_token_ids=new_token_ids,
106
+ prompt=new_prompt,
107
+ multi_modal_data=multi_modal_data)
108
+
109
+
110
+# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
111
+class BlipVisionEmbeddings(nn.Module):
112
+
113
+ def __init__(self, config: BlipVisionConfig):
114
+ super().__init__()
115
+
116
+ self.config = config
117
+ self.embed_dim = config.hidden_size
118
+ self.image_size = config.image_size
119
+ self.patch_size = config.patch_size
120
+
121
+ self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim))
122
+
123
+ self.patch_embedding = nn.Conv2d(
124
+ in_channels=3,
125
+ out_channels=self.embed_dim,
126
+ kernel_size=self.patch_size,
127
+ stride=self.patch_size,
128
+ )
129
+
130
+ self.num_patches = get_blip_num_patches(image_size=self.image_size,
131
+ patch_size=self.patch_size)
132
+ self.num_positions = self.num_patches + 1
133
+
134
+ self.position_embedding = nn.Parameter(
135
+ torch.randn(1, self.num_positions, self.embed_dim))
136
+
137
+ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
138
+ batch_size = pixel_values.shape[0]
139
+ target_dtype = self.patch_embedding.weight.dtype
140
+ patch_embeds = self.patch_embedding(pixel_values.to(
141
+ dtype=target_dtype)) # shape = [*, width, grid, grid]
142
+ patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
143
+
144
+ class_embeds = self.class_embedding.expand(batch_size, 1, -1)
145
+ embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
146
+
147
+ position_embeds = self.position_embedding.to(target_dtype)
148
+ embeddings = embeddings + position_embeds[:, :embeddings.size(1), :]
149
+
150
+ return embeddings
151
+
152
+
153
+class BlipMLP(nn.Module):
154
+
155
+ def __init__(self,
156
+ config: BlipVisionConfig,
157
+ quant_config: Optional[QuantizationConfig] = None):
158
+ super().__init__()
159
+
160
+ self.config = config
161
+
162
+ self.activation_fn = get_act_fn(config.hidden_act)
163
+ self.fc1 = ColumnParallelLinear(config.hidden_size,
164
+ config.intermediate_size,
165
+ bias=True,
166
+ quant_config=quant_config)
167
+ self.fc2 = RowParallelLinear(config.intermediate_size,
168
+ config.hidden_size,
169
+ bias=True,
170
+ quant_config=quant_config)
171
+
172
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
173
+ hidden_states, _ = self.fc1(hidden_states)
174
+ hidden_states = self.activation_fn(hidden_states)
175
+ hidden_states, _ = self.fc2(hidden_states)
176
+
177
+ return hidden_states
178
+
179
+
180
+class BlipEncoderLayer(nn.Module):
181
+
182
+ def __init__(self,
183
+ config: BlipVisionConfig,
184
+ quant_config: Optional[QuantizationConfig] = None):
185
+ super().__init__()
186
+
187
+ self.self_attn = BlipAttention(config)
188
+ self.layer_norm1 = nn.LayerNorm(config.hidden_size,
189
+ eps=config.layer_norm_eps)
190
+ self.mlp = BlipMLP(config, quant_config=quant_config)
191
+ self.layer_norm2 = nn.LayerNorm(config.hidden_size,
192
+ eps=config.layer_norm_eps)
193
+
194
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
195
+ residual = hidden_states
196
+
197
+ hidden_states = self.layer_norm1(hidden_states)
198
+ hidden_states, _ = self.self_attn(hidden_states=hidden_states)
199
+ hidden_states = residual + hidden_states
200
+
201
+ residual = hidden_states
202
+ hidden_states = self.layer_norm2(hidden_states)
203
+ hidden_states = self.mlp(hidden_states)
204
+ hidden_states = residual + hidden_states
205
+
206
+ return hidden_states
207
+
208
+
209
+class BlipEncoder(nn.Module):
210
+ """
211
+ Transformer encoder consisting of `config.num_hidden_layers` self
212
+ attention layers. Each layer is a [`BlipEncoderLayer`].
213
+
214
+ Args:
215
+ config: BlipConfig
216
+ """
217
+
218
+ def __init__(self,
219
+ config: BlipVisionConfig,
220
+ quant_config: Optional[QuantizationConfig] = None,
221
+ num_hidden_layers_override: Optional[int] = None):
222
+ super().__init__()
223
+
224
+ self.config = config
225
+
226
+ if num_hidden_layers_override is None:
227
+ num_hidden_layers = config.num_hidden_layers
228
+ else:
229
+ num_hidden_layers = num_hidden_layers_override
230
+
231
+ self.layers = nn.ModuleList([
232
+ BlipEncoderLayer(config=config, quant_config=quant_config)
233
+ for _ in range(num_hidden_layers)
234
+ ])
235
+
236
+ def forward(self, inputs_embeds: torch.Tensor):
237
+ hidden_states = inputs_embeds
238
+ for encoder_layer in self.layers:
239
+ hidden_states = encoder_layer(hidden_states)
240
+
241
+ return hidden_states
242
+
243
+
244
+class BlipVisionModel(nn.Module):
245
+ config_class = BlipVisionConfig
246
+ main_input_name = "pixel_values"
247
+
248
+ def __init__(self,
249
+ config: BlipVisionConfig,
250
+ quant_config: Optional[QuantizationConfig] = None,
251
+ num_hidden_layers_override: Optional[int] = None):
252
+ super().__init__()
253
+
254
+ self.config = config
255
+
256
+ self.embeddings = BlipVisionEmbeddings(config)
257
+ self.encoder = BlipEncoder(
258
+ config=config,
259
+ quant_config=quant_config,
260
+ num_hidden_layers_override=num_hidden_layers_override,
261
+ )
262
+ self.post_layernorm = nn.LayerNorm(config.hidden_size,
263
+ eps=config.layer_norm_eps)
264
+
265
+ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
266
+ hidden_states = self.embeddings(pixel_values)
267
+ hidden_states = self.encoder(inputs_embeds=hidden_states)
268
+
269
+ return self.post_layernorm(hidden_states)
vllm/model_executor/models/blip2.py
new
+669
@@ -0,0 +1,669 @@
1
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
2
+
3
+import torch
4
+import torch.nn as nn
5
+from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
6
+ apply_chunking_to_forward)
7
+
8
+from vllm.attention import AttentionMetadata
9
+from vllm.config import CacheConfig, MultiModalConfig
10
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11
+from vllm.model_executor.layers.activation import get_act_fn
12
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
13
+from vllm.model_executor.layers.quantization import QuantizationConfig
14
+from vllm.model_executor.layers.sampler import Sampler
15
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
+from vllm.model_executor.models.opt import OPTModel
17
+from vllm.model_executor.sampling_metadata import SamplingMetadata
18
+from vllm.multimodal import MULTIMODAL_REGISTRY
19
+from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
20
+
21
+from .blip import (BlipVisionModel, dummy_image_for_blip,
22
+ get_max_blip_image_tokens)
23
+from .interfaces import SupportsVision
24
+from .utils import merge_vision_embeddings
25
+
26
+_KEYS_TO_MODIFY_MAPPING = {
27
+ "language_model.lm_head": "lm_head",
28
+ "language_model.model": "language_model",
29
+}
30
+
31
+
32
+class Blip2QFormerMultiHeadAttention(nn.Module):
33
+
34
+ def __init__(
35
+ self,
36
+ config: Blip2QFormerConfig,
37
+ *,
38
+ quant_config: Optional[QuantizationConfig],
39
+ cache_config: Optional[CacheConfig],
40
+ is_cross_attention: bool = False,
41
+ ) -> None:
42
+ super().__init__()
43
+
44
+ self.config = config
45
+
46
+ if config.hidden_size % config.num_attention_heads != 0:
47
+ raise ValueError(
48
+ f"The hidden size ({config.hidden_size}) is not a multiple of "
49
+ f"the number of attention heads ({config.num_attention_heads})"
50
+ )
51
+
52
+ self.num_attention_heads = config.num_attention_heads
53
+ self.attention_head_size = (config.hidden_size //
54
+ config.num_attention_heads)
55
+ self.all_head_size = self.num_attention_heads * self.attention_head_size
56
+ self.scaling = self.attention_head_size**-0.5
57
+
58
+ self.query = nn.Linear(config.hidden_size, self.all_head_size)
59
+ if is_cross_attention:
60
+ kv_hidden_size = config.encoder_hidden_size
61
+ else:
62
+ kv_hidden_size = config.hidden_size
63
+ self.key = nn.Linear(kv_hidden_size, self.all_head_size)
64
+ self.value = nn.Linear(kv_hidden_size, self.all_head_size)
65
+
66
+ self.position_embedding_type = getattr(config,
67
+ "position_embedding_type",
68
+ "absolute")
69
+ if self.position_embedding_type != "absolute":
70
+ raise NotImplementedError("Unsupported position_embedding_type: "
71
+ f"{self.position_embedding_type}")
72
+
73
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
74
+
75
+ def transpose_for_scores(self, x):
76
+ x = x.view(*x.size()[:-1], self.num_attention_heads,
77
+ self.attention_head_size)
78
+ return x.permute(0, 2, 1, 3)
79
+
80
+ def forward(
81
+ self,
82
+ hidden_states: torch.Tensor,
83
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
84
+ ):
85
+ is_cross_attention = encoder_hidden_states is not None
86
+
87
+ if is_cross_attention:
88
+ key_layer = self.transpose_for_scores(
89
+ self.key(encoder_hidden_states))
90
+ value_layer = self.transpose_for_scores(
91
+ self.value(encoder_hidden_states))
92
+ else:
93
+ key_layer = self.transpose_for_scores(self.key(hidden_states))
94
+ value_layer = self.transpose_for_scores(self.value(hidden_states))
95
+
96
+ mixed_query_layer = self.query(hidden_states)
97
+
98
+ query_layer = self.transpose_for_scores(mixed_query_layer)
99
+
100
+ attention_scores = torch.matmul(query_layer,
101
+ key_layer.transpose(-1, -2))
102
+ attention_probs = torch.softmax(attention_scores * self.scaling,
103
+ dim=-1)
104
+
105
+ # This is actually dropping out entire tokens to attend to, which might
106
+ # seem a bit unusual, but is taken from the original Transformer paper.
107
+ attention_probs_dropped = self.dropout(attention_probs)
108
+
109
+ context_layer = torch.matmul(attention_probs_dropped, value_layer)
110
+
111
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
112
+ context_layer = context_layer.view(*context_layer.size()[:-2],
113
+ self.all_head_size)
114
+
115
+ return context_layer
116
+
117
+
118
+class Blip2QFormerSelfOutput(nn.Module):
119
+
120
+ def __init__(self, config: Blip2QFormerConfig) -> None:
121
+ super().__init__()
122
+
123
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
124
+ self.LayerNorm = nn.LayerNorm(config.hidden_size,
125
+ eps=config.layer_norm_eps)
126
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
127
+
128
+ def forward(
129
+ self,
130
+ hidden_states: torch.Tensor,
131
+ input_tensor: torch.Tensor,
132
+ ) -> torch.Tensor:
133
+ hidden_states = self.dense(hidden_states)
134
+ hidden_states = self.dropout(hidden_states)
135
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
136
+ return hidden_states
137
+
138
+
139
+class Blip2QFormerAttention(nn.Module):
140
+
141
+ def __init__(
142
+ self,
143
+ config: Blip2QFormerConfig,
144
+ *,
145
+ quant_config: Optional[QuantizationConfig],
146
+ cache_config: Optional[CacheConfig],
147
+ is_cross_attention: bool = False,
148
+ ) -> None:
149
+ super().__init__()
150
+
151
+ self.attention = Blip2QFormerMultiHeadAttention(
152
+ config,
153
+ quant_config=quant_config,
154
+ cache_config=cache_config,
155
+ is_cross_attention=is_cross_attention,
156
+ )
157
+
158
+ self.output = Blip2QFormerSelfOutput(config)
159
+
160
+ def forward(
161
+ self,
162
+ hidden_states: torch.Tensor,
163
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
164
+ ) -> Tuple[torch.Tensor]:
165
+ self_output = self.attention(
166
+ hidden_states,
167
+ encoder_hidden_states=encoder_hidden_states,
168
+ )
169
+ attention_output = self.output(self_output, hidden_states)
170
+
171
+ return attention_output
172
+
173
+
174
+class Blip2QFormerIntermediate(nn.Module):
175
+
176
+ def __init__(self, config: Blip2QFormerConfig) -> None:
177
+ super().__init__()
178
+
179
+ self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
180
+ self.intermediate_act_fn = get_act_fn(config.hidden_act)
181
+
182
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
183
+ hidden_states = self.dense(hidden_states)
184
+ hidden_states = self.intermediate_act_fn(hidden_states)
185
+ return hidden_states
186
+
187
+
188
+class Blip2QFormerOutput(nn.Module):
189
+
190
+ def __init__(self, config: Blip2QFormerConfig) -> None:
191
+ super().__init__()
192
+
193
+ self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
194
+ self.LayerNorm = nn.LayerNorm(config.hidden_size,
195
+ eps=config.layer_norm_eps)
196
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
197
+
198
+ def forward(
199
+ self,
200
+ hidden_states: torch.Tensor,
201
+ input_tensor: torch.Tensor,
202
+ ) -> torch.Tensor:
203
+ hidden_states = self.dense(hidden_states)
204
+ hidden_states = self.dropout(hidden_states)
205
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
206
+ return hidden_states
207
+
208
+
209
+class Blip2QFormerLayer(nn.Module):
210
+
211
+ def __init__(
212
+ self,
213
+ config: Blip2QFormerConfig,
214
+ *,
215
+ quant_config: Optional[QuantizationConfig],
216
+ cache_config: Optional[CacheConfig],
217
+ layer_idx: int,
218
+ ) -> None:
219
+ super().__init__()
220
+
221
+ self.chunk_size_feed_forward = config.chunk_size_feed_forward
222
+ self.seq_len_dim = 1
223
+ self.attention = Blip2QFormerAttention(config,
224
+ quant_config=quant_config,
225
+ cache_config=cache_config)
226
+
227
+ self.layer_idx = layer_idx
228
+
229
+ if layer_idx % config.cross_attention_frequency == 0:
230
+ self.crossattention = Blip2QFormerAttention(
231
+ config,
232
+ quant_config=quant_config,
233
+ cache_config=cache_config,
234
+ is_cross_attention=True)
235
+ self.has_cross_attention = True
236
+ else:
237
+ self.has_cross_attention = False
238
+
239
+ self.intermediate_query = Blip2QFormerIntermediate(config)
240
+ self.output_query = Blip2QFormerOutput(config)
241
+
242
+ def forward(
243
+ self,
244
+ hidden_states: torch.FloatTensor,
245
+ encoder_hidden_states: torch.FloatTensor,
246
+ query_length: int,
247
+ ):
248
+ attention_output = self.attention(hidden_states)
249
+
250
+ if query_length > 0:
251
+ query_attention_output = attention_output[:, :query_length, :]
252
+
253
+ if self.has_cross_attention:
254
+ query_attention_output = self.crossattention(
255
+ query_attention_output,
256
+ encoder_hidden_states=encoder_hidden_states,
257
+ )
258
+
259
+ layer_output = apply_chunking_to_forward(
260
+ self.feed_forward_chunk_query,
261
+ self.chunk_size_feed_forward,
262
+ self.seq_len_dim,
263
+ query_attention_output,
264
+ )
265
+
266
+ if attention_output.shape[1] > query_length:
267
+ layer_output_text = apply_chunking_to_forward(
268
+ self.feed_forward_chunk,
269
+ self.chunk_size_feed_forward,
270
+ self.seq_len_dim,
271
+ attention_output[:, query_length:, :],
272
+ )
273
+ layer_output = torch.cat([layer_output, layer_output_text],
274
+ dim=1)
275
+ else:
276
+ layer_output = apply_chunking_to_forward(
277
+ self.feed_forward_chunk,
278
+ self.chunk_size_feed_forward,
279
+ self.seq_len_dim,
280
+ attention_output,
281
+ )
282
+
283
+ return layer_output
284
+
285
+ def feed_forward_chunk(self,
286
+ attention_output: torch.Tensor) -> torch.Tensor:
287
+ intermediate_output = self.intermediate(attention_output)
288
+ layer_output = self.output(intermediate_output, attention_output)
289
+ return layer_output
290
+
291
+ def feed_forward_chunk_query(
292
+ self, attention_output: torch.Tensor) -> torch.Tensor:
293
+ intermediate_output = self.intermediate_query(attention_output)
294
+ layer_output = self.output_query(intermediate_output, attention_output)
295
+ return layer_output
296
+
297
+
298
+class Blip2QFormerEncoder(nn.Module):
299
+
300
+ def __init__(
301
+ self,
302
+ config: Blip2QFormerConfig,
303
+ *,
304
+ quant_config: Optional[QuantizationConfig],
305
+ cache_config: Optional[CacheConfig],
306
+ ) -> None:
307
+ super().__init__()
308
+
309
+ self.config = config
310
+
311
+ self.layer = nn.ModuleList([
312
+ Blip2QFormerLayer(config,
313
+ quant_config=quant_config,
314
+ cache_config=cache_config,
315
+ layer_idx=layer_idx)
316
+ for layer_idx in range(config.num_hidden_layers)
317
+ ])
318
+
319
+ def forward(
320
+ self,
321
+ hidden_states: torch.FloatTensor,
322
+ encoder_hidden_states: torch.FloatTensor,
323
+ query_length: int,
324
+ ) -> torch.Tensor:
325
+ for i in range(self.config.num_hidden_layers):
326
+ layer_module = self.layer[i]
327
+
328
+ hidden_states = layer_module(
329
+ hidden_states,
330
+ encoder_hidden_states=encoder_hidden_states,
331
+ query_length=query_length,
332
+ )
333
+
334
+ return hidden_states
335
+
336
+
337
+# Adapted from https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1025
338
+class Blip2QFormerModel(nn.Module):
339
+
340
+ def __init__(
341
+ self,
342
+ config: Blip2QFormerConfig,
343
+ *,
344
+ quant_config: Optional[QuantizationConfig],
345
+ cache_config: Optional[CacheConfig],
346
+ ) -> None:
347
+ super().__init__()
348
+
349
+ self.config = config
350
+
351
+ self.layernorm = nn.LayerNorm(config.hidden_size,
352
+ eps=config.layer_norm_eps)
353
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
354
+
355
+ self.encoder = Blip2QFormerEncoder(config,
356
+ quant_config=quant_config,
357
+ cache_config=cache_config)
358
+
359
+ def forward(
360
+ self,
361
+ query_embeds: torch.FloatTensor,
362
+ encoder_hidden_states: torch.FloatTensor,
363
+ ) -> torch.Tensor:
364
+ query_length = query_embeds.shape[1]
365
+
366
+ embedding_output = self.layernorm(query_embeds)
367
+ embedding_output = self.dropout(embedding_output)
368
+
369
+ sequence_output = self.encoder(
370
+ embedding_output,
371
+ encoder_hidden_states=encoder_hidden_states,
372
+ query_length=query_length,
373
+ )
374
+
375
+ return sequence_output
376
+
377
+
378
+class Blip2ImagePixelInputs(TypedDict):
379
+ type: Literal["pixel_values"]
380
+ data: torch.Tensor
381
+ """Shape: (batch_size, num_channels, height, width)"""
382
+
383
+
384
+Blip2ImageInputs = Blip2ImagePixelInputs
385
+
386
+# We use this internally as placeholders since there is no image token
387
+# defined on the HuggingFace repo
388
+BLIP2_IMAGE_TOKEN = "<image>"
389
+BLIP2_IMAGE_TOKEN_ID = 50265
390
+
391
+
392
+def get_blip2_image_feature_size(hf_config: Blip2Config) -> int:
393
+ return hf_config.num_query_tokens
394
+
395
+
396
+def get_max_blip2_image_tokens(ctx: InputContext):
397
+ hf_config = ctx.get_hf_config(Blip2Config)
398
+ vision_config = hf_config.vision_config
399
+
400
+ if isinstance(vision_config, Blip2VisionConfig):
401
+ return get_max_blip_image_tokens(vision_config)
402
+
403
+ msg = f"Unsupported vision config: {type(vision_config)}"
404
+ raise NotImplementedError(msg)
405
+
406
+
407
+def dummy_data_for_blip2(ctx: InputContext, seq_len: int):
408
+ hf_config = ctx.get_hf_config(Blip2Config)
409
+ vision_config = hf_config.vision_config
410
+
411
+ image_feature_size = get_blip2_image_feature_size(hf_config)
412
+ token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
413
+ token_ids += [0] * (seq_len - image_feature_size)
414
+ seq_data = SequenceData(token_ids)
415
+
416
+ if isinstance(vision_config, Blip2VisionConfig):
417
+ mm_data = dummy_image_for_blip(vision_config)
418
+
419
+ return seq_data, mm_data
420
+
421
+ msg = f"Unsupported vision config: {type(vision_config)}"
422
+ raise NotImplementedError(msg)
423
+
424
+
425
+def input_processor_for_blip2(ctx: InputContext, llm_inputs: LLMInputs):
426
+ multi_modal_data = llm_inputs.get("multi_modal_data")
427
+ if multi_modal_data is None or "image" not in multi_modal_data:
428
+ return llm_inputs
429
+
430
+ hf_config = ctx.get_hf_config(Blip2Config)
431
+ image_feature_size = get_blip2_image_feature_size(hf_config)
432
+
433
+ # The original model places image tokens at the front
434
+ # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1514
435
+ new_token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
436
+ new_token_ids += llm_inputs["prompt_token_ids"]
437
+
438
+ new_prompt = llm_inputs.get("prompt")
439
+ if new_prompt is not None:
440
+ new_prompt = BLIP2_IMAGE_TOKEN * image_feature_size + new_prompt
441
+
442
+ return LLMInputs(prompt_token_ids=new_token_ids,
443
+ prompt=new_prompt,
444
+ multi_modal_data=multi_modal_data)
445
+
446
+
447
+@MULTIMODAL_REGISTRY.register_image_input_mapper()
448
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_blip2_image_tokens)
449
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_blip2)
450
+@INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
451
+class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
452
+
453
+ def __init__(self,
454
+ config: Blip2Config,
455
+ multimodal_config: MultiModalConfig,
456
+ cache_config: Optional[CacheConfig] = None,
457
+ quant_config: Optional[QuantizationConfig] = None) -> None:
458
+
459
+ super().__init__()
460
+
461
+ self.config = config
462
+ self.multimodal_config = multimodal_config
463
+
464
+ # TODO: Optionally initializes this for supporting embeddings.
465
+ self.vision_model = BlipVisionModel(config.vision_config)
466
+
467
+ self.query_tokens = nn.Parameter(
468
+ torch.zeros(1, config.num_query_tokens,
469
+ config.qformer_config.hidden_size))
470
+
471
+ self.qformer = Blip2QFormerModel(config.qformer_config,
472
+ cache_config=cache_config,
473
+ quant_config=quant_config)
474
+
475
+ self.language_projection = nn.Linear(
476
+ config.qformer_config.hidden_size,
477
+ config.text_config.hidden_size,
478
+ bias=True,
479
+ )
480
+
481
+ self.quant_config = quant_config
482
+
483
+ self.language_model = OPTModel(config.text_config, cache_config,
484
+ quant_config)
485
+
486
+ self.unpadded_vocab_size = config.text_config.vocab_size
487
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size)
488
+ self.sampler = Sampler()
489
+
490
+ def get_lm_head(self):
491
+ return self.language_model.decoder.embed_tokens
492
+
493
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
494
+ h = w = self.config.vision_config.image_size
495
+ expected_dims = (3, h, w)
496
+ actual_dims = tuple(data.shape[1:])
497
+
498
+ if actual_dims != expected_dims:
499
+ expected_expr = ("batch_size", *map(str, expected_dims))
500
+ raise ValueError(
501
+ f"The expected shape of pixel values is {expected_expr}. "
502
+ f"You supplied {tuple(data.shape)}.")
503
+
504
+ return data
505
+
506
+ def _parse_and_validate_image_input(
507
+ self, **kwargs: object) -> Optional[Blip2ImageInputs]:
508
+ pixel_values = kwargs.pop("pixel_values", None)
509
+
510
+ if pixel_values is None:
511
+ return None
512
+
513
+ if not isinstance(pixel_values, torch.Tensor):
514
+ raise ValueError("Incorrect type of pixel values. "
515
+ f"Got type: {type(pixel_values)}")
516
+
517
+ return Blip2ImagePixelInputs(
518
+ type="pixel_values",
519
+ data=self._validate_pixel_values(pixel_values),
520
+ )
521
+
522
+ def _image_pixels_to_features(self, vision_model: BlipVisionModel,
523
+ pixel_values: torch.Tensor) -> torch.Tensor:
524
+
525
+ # NOTE: we skip the step to select the vision feature layer since
526
+ # this is already done inside the vision tower
527
+ image_features = vision_model(pixel_values)
528
+
529
+ return image_features
530
+
531
+ def _process_image_pixels(self,
532
+ inputs: Blip2ImagePixelInputs) -> torch.Tensor:
533
+ assert self.vision_model is not None
534
+
535
+ pixel_values = inputs["data"]
536
+
537
+ return self._image_pixels_to_features(self.vision_model, pixel_values)
538
+
539
+ def _process_image_input(self,
540
+ image_input: Blip2ImageInputs) -> torch.Tensor:
541
+ assert self.vision_model is not None
542
+ image_features = self._process_image_pixels(image_input)
543
+
544
+ query_tokens = self.query_tokens.expand(image_features.shape[0], -1,
545
+ -1)
546
+ query_output = self.qformer(
547
+ query_embeds=query_tokens,
548
+ encoder_hidden_states=image_features,
549
+ )
550
+
551
+ return self.language_projection(query_output)
552
+
553
+ def forward(
554
+ self,
555
+ input_ids: torch.Tensor,
556
+ positions: torch.Tensor,
557
+ kv_caches: List[torch.Tensor],
558
+ attn_metadata: AttentionMetadata,
559
+ intermediate_tensors: Optional[IntermediateTensors] = None,
560
+ **kwargs: object,
561
+ ) -> SamplerOutput:
562
+ """Run forward pass for BLIP-2.
563
+
564
+ One key thing to understand is the `input_ids` already accounts for the
565
+ positions of the to-be-inserted image embeddings.
566
+
567
+ Concretely, consider a text prompt:
568
+ `"Question: What's the content of the image? Answer:"`.
569
+
570
+ Tokenizer outputs:
571
+ `[2, 45641, 35, 653, 18, 5, 1383, 9, 5, 2274, 116, 31652, 35]`.
572
+
573
+ To reserve space in KV cache, we have to insert placeholder tokens
574
+ before they are inputted to the model, so the input processor prepends
575
+ dummy tokens (denoted as `50265`), resulting in:
576
+ `[50265, ..., 50265, 2, 45641, 35, ..., 31652, 35]`.
577
+
578
+ We insert 32 tokens since it corresponds to the number of query
579
+ embeddings outputted by the Q-Former and inputted to the language model.
580
+
581
+ This way, the `positions` and `attn_metadata` are consistent
582
+ with the `input_ids`.
583
+
584
+ Args:
585
+ input_ids: Flattened (concatenated) input_ids corresponding to a
586
+ batch.
587
+ pixel_values: The pixels in each input image.
588
+
589
+ See also:
590
+ :class:`Blip2ImageInputs`
591
+ """
592
+ image_input = self._parse_and_validate_image_input(**kwargs)
593
+
594
+ if image_input is not None:
595
+ vision_embeddings = self._process_image_input(image_input)
596
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
597
+
598
+ inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
599
+ vision_embeddings,
600
+ BLIP2_IMAGE_TOKEN_ID)
601
+
602
+ input_ids = None
603
+ else:
604
+ inputs_embeds = None
605
+
606
+ hidden_states = self.language_model(input_ids,
607
+ positions,
608
+ kv_caches,
609
+ attn_metadata,
610
+ inputs_embeds=inputs_embeds)
611
+
612
+ return hidden_states
613
+
614
+ def compute_logits(self, hidden_states: torch.Tensor,
615
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
616
+ logits = self.logits_processor(self.get_lm_head(), hidden_states,
617
+ sampling_metadata)
618
+ return logits
619
+
620
+ def sample(
621
+ self,
622
+ logits: torch.Tensor,
623
+ sampling_metadata: SamplingMetadata,
624
+ ) -> Optional[SamplerOutput]:
625
+ next_tokens = self.sampler(logits, sampling_metadata)
626
+ return next_tokens
627
+
628
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
629
+ # only doing this for language model part for now.
630
+ stacked_params_mapping = [
631
+ # (param_name, shard_name, shard_id)
632
+ ("qkv_proj", "q_proj", "q"),
633
+ ("qkv_proj", "k_proj", "k"),
634
+ ("qkv_proj", "v_proj", "v"),
635
+ ("gate_up_proj", "gate_proj", 0),
636
+ ("gate_up_proj", "up_proj", 1),
637
+ ]
638
+ params_dict = dict(self.named_parameters())
639
+
640
+ for name, loaded_weight in weights:
641
+ if "lm_head.weight" in name:
642
+ continue
643
+ if "rotary_emb.inv_freq" in name:
644
+ continue
645
+ for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
646
+ if key_to_modify in name:
647
+ name = name.replace(key_to_modify, new_key)
648
+ use_default_weight_loading = False
649
+ if "vision" in name:
650
+ if self.vision_model is not None:
651
+ # We only do sharding for language model and
652
+ # not vision model for now.
653
+ use_default_weight_loading = True
654
+ else:
655
+ for (param_name, weight_name,
656
+ shard_id) in stacked_params_mapping:
657
+ if weight_name not in name:
658
+ continue
659
+ param = params_dict[name.replace(weight_name, param_name)]
660
+ weight_loader = param.weight_loader
661
+ weight_loader(param, loaded_weight, shard_id)
662
+ break
663
+ else:
664
+ use_default_weight_loading = True
665
+ if use_default_weight_loading:
666
+ param = params_dict[name]
667
+ weight_loader = getattr(param, "weight_loader",
668
+ default_weight_loader)
669
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/opt.py
+15
-2
@@ -237,14 +237,19 @@ class OPTDecoder(nn.Module):
237
for _ in range(config.num_hidden_layers)
238
])
239
240
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
241
+ return self.embed_tokens(input_ids)
242
+
243
def forward(
244
self,
245
input_ids: torch.Tensor,
246
positions: torch.Tensor,
247
kv_caches: List[torch.Tensor],
248
attn_metadata: AttentionMetadata,
249
+ inputs_embeds: Optional[torch.Tensor] = None,
250
) -> torch.Tensor:
247
- inputs_embeds = self.embed_tokens(input_ids)
251
+ if inputs_embeds is None:
252
+ inputs_embeds = self.get_input_embeddings(input_ids)
253
pos_embeds = self.embed_positions(positions)
254
if self.project_in is not None:
255
inputs_embeds, _ = self.project_in(inputs_embeds)
@@ -272,14 +277,22 @@ class OPTModel(nn.Module):
277
super().__init__()
278
self.decoder = OPTDecoder(config, cache_config, quant_config)
279
280
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
281
+ return self.decoder.get_input_embeddings(input_ids)
282
+
283
def forward(
284
self,
285
input_ids: torch.Tensor,
286
positions: torch.Tensor,
287
kv_caches: List[torch.Tensor],
288
attn_metadata: AttentionMetadata,
289
+ inputs_embeds: Optional[torch.Tensor] = None,
290
) -> torch.Tensor:
282
- return self.decoder(input_ids, positions, kv_caches, attn_metadata)
291
+ return self.decoder(input_ids,
292
+ positions,
293
+ kv_caches,
294
+ attn_metadata,
295
+ inputs_embeds=inputs_embeds)
296
297
298
class OPTForCausalLM(nn.Module):
vllm/multimodal/base.py
+6
-5
@@ -1,8 +1,9 @@
1
import sys
2
from abc import ABC, abstractmethod
3
from collections import UserDict, defaultdict
4
-from typing import (Any, Callable, Dict, List, Optional, Type, TypedDict,
5
- TypeVar, Union, cast)
4
+from typing import Any, Callable, Dict, List, Optional
5
+from typing import Sequence as GenericSequence
6
+from typing import Type, TypedDict, TypeVar, Union, cast
7
8
import torch
9
import torch.types
@@ -15,13 +16,13 @@ from vllm.logger import init_logger
16
17
logger = init_logger(__name__)
18
18
-NestedTensors = Union[List[torch.Tensor], torch.Tensor]
19
+NestedTensors = Union[GenericSequence[torch.Tensor], torch.Tensor]
20
"""
21
Use a list instead of a tensor if the dimensions of each element do not match.
22
Currently only supports up to singly nested list of tensors.
23
"""
24
24
-BatchedTensors = Union[List[NestedTensors], NestedTensors]
25
+BatchedTensors = Union[GenericSequence[NestedTensors], NestedTensors]
26
"""
27
If each input tensor in the batch has the same size, this is a single batched
28
tensor; otherwise, this is a list of :class:`NestedTensors` with one element
@@ -53,7 +54,7 @@ class MultiModalInputs(_MultiModalInputsBase):
54
# may be list rather than tensors
55
if isinstance(tensors[0], list):
56
return [[t.to(device=device) for t in tensor[0]]
56
- for tensor in tensors]
57
+ for tensor in cast(List[List[torch.Tensor]], tensors)]
58
59
tensors_ = cast(List[torch.Tensor], tensors)
60