@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ad86acf17
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model] Initial support for BLIP-2 (#5920)

Co-authored-by: ywang96 <ywang@roblox.com>

Cyrus Leung committed Jul 27, 2024 at 19:53 UTC 1ad86acf1789650e2ff27586e36a8159d52755dd
12 files changed +1107 -21
docs/source/models/supported_models.rst
+8
@@ -7,6 +7,8 @@ vLLM supports a variety of generative Transformer models in `HuggingFace Transfo
7 The following is the list of model architectures that are currently supported by vLLM.
8 Alongside each architecture, we include some popular models that use it.
9
10 +----
11 +
12 Decoder-only Language Models
13 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
14 .. list-table::
@@ -186,6 +188,10 @@ Vision Language Models
188 - Models
189 - Example HuggingFace Models
190 - :ref:`LoRA <lora>`
191 + * - :code:`Blip2ForConditionalGeneration`
192 + - BLIP-2
193 + - :code:`Salesforce/blip2-opt-2.7b`, :code:`Salesforce/blip2-opt-6.7b`, etc.
194 + -
195 * - :code:`ChameleonForConditionalGeneration`
196 - Chameleon
197 - :code:`facebook/chameleon-7b` etc.
@@ -215,6 +221,8 @@ Vision Language Models
221 - :code:`openbmb/MiniCPM-V-2`, :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
222 -
223
224 +----
225 +
226 If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
227 Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` and :ref:`Enabling Multimodal Inputs <enabling_multimodal_inputs>`
228 for instructions on how to implement support for your model.
examples/offline_inference_vision_language.py
+11
@@ -106,6 +106,16 @@ def run_minicpmv(question):
106 return llm, prompt
107
108
109 +# BLIP-2
110 +def run_blip2(question):
111 +
112 + # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
113 + # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
114 + prompt = f"Question: {question} Answer:"
115 + llm = LLM(model="Salesforce/blip2-opt-2.7b")
116 + return llm, prompt
117 +
118 +
119 model_example_map = {
120 "llava": run_llava,
121 "llava-next": run_llava_next,
@@ -114,6 +124,7 @@ model_example_map = {
124 "paligemma": run_paligemma,
125 "chameleon": run_chameleon,
126 "minicpmv": run_minicpmv,
127 + "blip-2": run_blip2,
128 }
129
130
examples/template_blip2.jinja new
+11
@@ -0,0 +1,11 @@
1 +{%- for message in messages -%}
2 + {%- if message['role'] == 'user' -%}
3 + {{- 'Question: ' + message['content'] + ' ' -}}
4 + {%- elif message['role'] == 'assistant' -%}
5 + {{- 'Answer: ' + message['content'] + ' ' -}}
6 + {%- endif -%}
7 +{%- endfor -%}
8 +
9 +{%- if add_generation_prompt -%}
10 + {{- 'Answer:' -}}
11 +{% endif %}
tests/models/test_blip2.py new
+102
@@ -0,0 +1,102 @@
1 +from typing import List, Optional, Tuple
2 +
3 +import pytest
4 +from transformers import AutoTokenizer
5 +
6 +from vllm.multimodal.utils import rescale_image_size
7 +from vllm.sequence import SampleLogprobs
8 +
9 +from ..conftest import IMAGE_ASSETS
10 +from .utils import check_logprobs_close
11 +
12 +pytestmark = pytest.mark.vlm
13 +
14 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
15 + "stop_sign":
16 + "Question: What's the content of the image? Answer:",
17 + "cherry_blossom":
18 + "Question: What is the season? Answer:",
19 +})
20 +
21 +
22 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
23 + Optional[SampleLogprobs]],
24 + model: str):
25 + """Sanitize vllm output to be comparable with hf output."""
26 + _, output_str, out_logprobs = vllm_output
27 +
28 + hf_output_str = output_str + "\n"
29 +
30 + tokenizer = AutoTokenizer.from_pretrained(model)
31 + hf_output_ids = tokenizer.encode(hf_output_str)
32 + assert hf_output_ids[0] == tokenizer.bos_token_id
33 + hf_output_ids = hf_output_ids[1:]
34 +
35 + return hf_output_ids, hf_output_str, out_logprobs
36 +
37 +
38 +@pytest.mark.parametrize("model", ["Salesforce/blip2-opt-2.7b"])
39 +@pytest.mark.parametrize(
40 + "size_factors",
41 + [
42 + # No image
43 + [],
44 + # Single-scale
45 + [1.0],
46 + # Single-scale, batched
47 + [1.0, 1.0, 1.0],
48 + # Multi-scale
49 + [0.25, 0.5, 1.0],
50 + ],
51 +)
52 +@pytest.mark.parametrize("dtype", ["half"])
53 +@pytest.mark.parametrize("max_tokens", [128])
54 +@pytest.mark.parametrize("num_logprobs", [5])
55 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
56 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
57 + """Inference result should be the same between hf and vllm.
58 +
59 + All the image fixtures for the test is under tests/images.
60 + For huggingface runner, we provide the PIL images as input.
61 + For vllm runner, we provide MultiModalData objects and corresponding
62 + vision language config as input.
63 + Note, the text input is also adjusted to abide by vllm contract.
64 + The text output is sanitized to be able to compare with hf.
65 + """
66 + images = [asset.pil_image for asset in image_assets]
67 +
68 + inputs_per_image = [(
69 + [prompt for _ in size_factors],
70 + [rescale_image_size(image, factor) for factor in size_factors],
71 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
72 +
73 + # max_model_len should be greater than image_feature_size
74 + with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
75 + vllm_outputs_per_image = [
76 + vllm_model.generate_greedy_logprobs(prompts,
77 + max_tokens,
78 + num_logprobs=num_logprobs,
79 + images=images)
80 + for prompts, images in inputs_per_image
81 + ]
82 +
83 + with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
84 + hf_outputs_per_image = [
85 + hf_model.generate_greedy_logprobs_limit(prompts,
86 + max_tokens,
87 + num_logprobs=num_logprobs,
88 + images=images)
89 + for prompts, images in inputs_per_image
90 + ]
91 +
92 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
93 + vllm_outputs_per_image):
94 + check_logprobs_close(
95 + outputs_0_lst=hf_outputs,
96 + outputs_1_lst=[
97 + vllm_to_hf_output(vllm_output, model)
98 + for vllm_output in vllm_outputs
99 + ],
100 + name_0="hf",
101 + name_1="vllm",
102 + )
tests/models/test_fuyu.py
+4 -4
@@ -77,8 +77,8 @@ def run_test(
77 vllm_model.generate_greedy_logprobs(prompts,
78 max_tokens,
79 num_logprobs=num_logprobs,
80 - images=vllm_images)
81 - for prompts, vllm_images in inputs_per_image
80 + images=images)
81 + for prompts, images in inputs_per_image
82 ]
83
84 with hf_runner(model, dtype=dtype) as hf_model:
@@ -89,9 +89,9 @@ def run_test(
89 hf_model.generate_greedy_logprobs_limit(prompts,
90 max_tokens,
91 num_logprobs=num_logprobs,
92 - images=hf_images,
92 + images=images,
93 eos_token_id=eos_token_id)
94 - for prompts, hf_images in inputs_per_image
94 + for prompts, images in inputs_per_image
95 ]
96
97 for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
tests/models/test_minicpmv.py
+4 -4
@@ -88,9 +88,9 @@ def run_test(
88 vllm_model.generate_greedy_logprobs(prompts,
89 max_tokens,
90 num_logprobs=num_logprobs,
91 - images=vllm_images,
91 + images=images,
92 stop_token_ids=stop_token_ids)
93 - for prompts, vllm_images in inputs_per_image
93 + for prompts, images in inputs_per_image
94 ]
95
96 with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
@@ -114,9 +114,9 @@ def run_test(
114 hf_model.generate_greedy_logprobs_limit(prompts,
115 max_tokens,
116 num_logprobs=num_logprobs,
117 - images=hf_images,
117 + images=images,
118 tokenizer=tokenizer)
119 - for prompts, hf_images in inputs_per_image
119 + for prompts, images in inputs_per_image
120 ]
121
122 for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
tests/models/test_phi3v.py
+4 -4
@@ -101,8 +101,8 @@ def run_test(
101 vllm_model.generate_greedy_logprobs(prompts,
102 max_tokens,
103 num_logprobs=num_logprobs,
104 - images=vllm_images)
105 - for prompts, vllm_images in inputs_per_image
104 + images=images)
105 + for prompts, images in inputs_per_image
106 ]
107
108 # use eager mode for hf runner, since phi3_v didn't work with flash_attn
@@ -114,9 +114,9 @@ def run_test(
114 hf_model.generate_greedy_logprobs_limit(prompts,
115 max_tokens,
116 num_logprobs=num_logprobs,
117 - images=hf_images,
117 + images=images,
118 eos_token_id=eos_token_id)
119 - for prompts, hf_images in inputs_per_image
119 + for prompts, images in inputs_per_image
120 ]
121
122 for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
vllm/model_executor/models/__init__.py
+4 -2
@@ -16,6 +16,8 @@ _GENERATION_MODELS = {
16 "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
17 "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
18 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
19 + "Blip2ForConditionalGeneration":
20 + ("blip2", "Blip2ForConditionalGeneration"),
21 "ChameleonForConditionalGeneration":
22 ("chameleon", "ChameleonForConditionalGeneration"),
23 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
@@ -56,8 +58,8 @@ _GENERATION_MODELS = {
58 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
59 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
60 "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
59 - "PaliGemmaForConditionalGeneration":
60 - ("paligemma", "PaliGemmaForConditionalGeneration"),
61 + "PaliGemmaForConditionalGeneration": ("paligemma",
62 + "PaliGemmaForConditionalGeneration"),
63 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
64 "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
65 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
vllm/model_executor/models/blip.py new
+269
@@ -0,0 +1,269 @@
1 +"""Minimal implementation of BlipVisionModel intended to be only used
2 +within a vision language model."""
3 +from typing import Optional, Union
4 +
5 +import torch
6 +import torch.nn as nn
7 +from PIL import Image
8 +from transformers import Blip2VisionConfig, BlipVisionConfig
9 +from transformers.models.blip.modeling_blip import BlipAttention
10 +
11 +from vllm.config import ModelConfig
12 +from vllm.inputs import LLMInputs
13 +from vllm.model_executor.layers.activation import get_act_fn
14 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
15 + RowParallelLinear)
16 +from vllm.model_executor.layers.quantization import QuantizationConfig
17 +from vllm.multimodal.image import (cached_get_tokenizer,
18 + repeat_and_pad_image_tokens)
19 +from vllm.sequence import SequenceData
20 +
21 +
22 +def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
23 + assert image_size % patch_size == 0
24 + return image_size // patch_size
25 +
26 +
27 +def get_blip_num_patches(*, image_size: int, patch_size: int) -> int:
28 + grid_length = get_blip_patch_grid_length(image_size=image_size,
29 + patch_size=patch_size)
30 + return grid_length * grid_length
31 +
32 +
33 +def get_blip_image_feature_size(
34 + hf_config: Union[BlipVisionConfig, Blip2VisionConfig], ) -> int:
35 + return get_blip_num_patches(image_size=hf_config.image_size,
36 + patch_size=hf_config.patch_size)
37 +
38 +
39 +def get_max_blip_image_tokens(
40 + hf_config: Union[BlipVisionConfig, Blip2VisionConfig], ) -> int:
41 + return get_blip_image_feature_size(hf_config)
42 +
43 +
44 +def dummy_seq_data_for_blip(
45 + hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
46 + seq_len: int,
47 + *,
48 + image_token_id: int,
49 + image_feature_size_override: Optional[int] = None,
50 +):
51 + if image_feature_size_override is None:
52 + image_feature_size = get_blip_image_feature_size(hf_config)
53 + else:
54 + image_feature_size = image_feature_size_override
55 +
56 + token_ids = [image_token_id] * image_feature_size
57 + token_ids += [0] * (seq_len - image_feature_size)
58 + return SequenceData(token_ids)
59 +
60 +
61 +def dummy_image_for_blip(
62 + hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
63 + *,
64 + image_width_override: Optional[int] = None,
65 + image_height_override: Optional[int] = None,
66 +):
67 + width = height = hf_config.image_size
68 + if image_width_override is not None:
69 + width = image_width_override
70 + if image_height_override is not None:
71 + height = image_height_override
72 +
73 + image = Image.new("RGB", (width, height), color=0)
74 + return {"image": image}
75 +
76 +
77 +def input_processor_for_blip(
78 + model_config: ModelConfig,
79 + hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
80 + llm_inputs: LLMInputs,
81 + *,
82 + image_token_id: int,
83 + image_feature_size_override: Optional[int] = None,
84 +):
85 + multi_modal_data = llm_inputs.get("multi_modal_data")
86 + if multi_modal_data is None or "image" not in multi_modal_data:
87 + return llm_inputs
88 +
89 + tokenizer = cached_get_tokenizer(model_config.tokenizer)
90 +
91 + if image_feature_size_override is None:
92 + image_feature_size = get_blip_image_feature_size(hf_config)
93 + else:
94 + image_feature_size = image_feature_size_override
95 +
96 + new_prompt, new_token_ids = repeat_and_pad_image_tokens(
97 + tokenizer,
98 + llm_inputs.get("prompt"),
99 + llm_inputs["prompt_token_ids"],
100 + image_token_id=image_token_id,
101 + repeat_count=image_feature_size,
102 + )
103 +
104 + # NOTE: Create a defensive copy of the original inputs
105 + return LLMInputs(prompt_token_ids=new_token_ids,
106 + prompt=new_prompt,
107 + multi_modal_data=multi_modal_data)
108 +
109 +
110 +# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
111 +class BlipVisionEmbeddings(nn.Module):
112 +
113 + def __init__(self, config: BlipVisionConfig):
114 + super().__init__()
115 +
116 + self.config = config
117 + self.embed_dim = config.hidden_size
118 + self.image_size = config.image_size
119 + self.patch_size = config.patch_size
120 +
121 + self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim))
122 +
123 + self.patch_embedding = nn.Conv2d(
124 + in_channels=3,
125 + out_channels=self.embed_dim,
126 + kernel_size=self.patch_size,
127 + stride=self.patch_size,
128 + )
129 +
130 + self.num_patches = get_blip_num_patches(image_size=self.image_size,
131 + patch_size=self.patch_size)
132 + self.num_positions = self.num_patches + 1
133 +
134 + self.position_embedding = nn.Parameter(
135 + torch.randn(1, self.num_positions, self.embed_dim))
136 +
137 + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
138 + batch_size = pixel_values.shape[0]
139 + target_dtype = self.patch_embedding.weight.dtype
140 + patch_embeds = self.patch_embedding(pixel_values.to(
141 + dtype=target_dtype)) # shape = [*, width, grid, grid]
142 + patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
143 +
144 + class_embeds = self.class_embedding.expand(batch_size, 1, -1)
145 + embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
146 +
147 + position_embeds = self.position_embedding.to(target_dtype)
148 + embeddings = embeddings + position_embeds[:, :embeddings.size(1), :]
149 +
150 + return embeddings
151 +
152 +
153 +class BlipMLP(nn.Module):
154 +
155 + def __init__(self,
156 + config: BlipVisionConfig,
157 + quant_config: Optional[QuantizationConfig] = None):
158 + super().__init__()
159 +
160 + self.config = config
161 +
162 + self.activation_fn = get_act_fn(config.hidden_act)
163 + self.fc1 = ColumnParallelLinear(config.hidden_size,
164 + config.intermediate_size,
165 + bias=True,
166 + quant_config=quant_config)
167 + self.fc2 = RowParallelLinear(config.intermediate_size,
168 + config.hidden_size,
169 + bias=True,
170 + quant_config=quant_config)
171 +
172 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
173 + hidden_states, _ = self.fc1(hidden_states)
174 + hidden_states = self.activation_fn(hidden_states)
175 + hidden_states, _ = self.fc2(hidden_states)
176 +
177 + return hidden_states
178 +
179 +
180 +class BlipEncoderLayer(nn.Module):
181 +
182 + def __init__(self,
183 + config: BlipVisionConfig,
184 + quant_config: Optional[QuantizationConfig] = None):
185 + super().__init__()
186 +
187 + self.self_attn = BlipAttention(config)
188 + self.layer_norm1 = nn.LayerNorm(config.hidden_size,
189 + eps=config.layer_norm_eps)
190 + self.mlp = BlipMLP(config, quant_config=quant_config)
191 + self.layer_norm2 = nn.LayerNorm(config.hidden_size,
192 + eps=config.layer_norm_eps)
193 +
194 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
195 + residual = hidden_states
196 +
197 + hidden_states = self.layer_norm1(hidden_states)
198 + hidden_states, _ = self.self_attn(hidden_states=hidden_states)
199 + hidden_states = residual + hidden_states
200 +
201 + residual = hidden_states
202 + hidden_states = self.layer_norm2(hidden_states)
203 + hidden_states = self.mlp(hidden_states)
204 + hidden_states = residual + hidden_states
205 +
206 + return hidden_states
207 +
208 +
209 +class BlipEncoder(nn.Module):
210 + """
211 + Transformer encoder consisting of `config.num_hidden_layers` self
212 + attention layers. Each layer is a [`BlipEncoderLayer`].
213 +
214 + Args:
215 + config: BlipConfig
216 + """
217 +
218 + def __init__(self,
219 + config: BlipVisionConfig,
220 + quant_config: Optional[QuantizationConfig] = None,
221 + num_hidden_layers_override: Optional[int] = None):
222 + super().__init__()
223 +
224 + self.config = config
225 +
226 + if num_hidden_layers_override is None:
227 + num_hidden_layers = config.num_hidden_layers
228 + else:
229 + num_hidden_layers = num_hidden_layers_override
230 +
231 + self.layers = nn.ModuleList([
232 + BlipEncoderLayer(config=config, quant_config=quant_config)
233 + for _ in range(num_hidden_layers)
234 + ])
235 +
236 + def forward(self, inputs_embeds: torch.Tensor):
237 + hidden_states = inputs_embeds
238 + for encoder_layer in self.layers:
239 + hidden_states = encoder_layer(hidden_states)
240 +
241 + return hidden_states
242 +
243 +
244 +class BlipVisionModel(nn.Module):
245 + config_class = BlipVisionConfig
246 + main_input_name = "pixel_values"
247 +
248 + def __init__(self,
249 + config: BlipVisionConfig,
250 + quant_config: Optional[QuantizationConfig] = None,
251 + num_hidden_layers_override: Optional[int] = None):
252 + super().__init__()
253 +
254 + self.config = config
255 +
256 + self.embeddings = BlipVisionEmbeddings(config)
257 + self.encoder = BlipEncoder(
258 + config=config,
259 + quant_config=quant_config,
260 + num_hidden_layers_override=num_hidden_layers_override,
261 + )
262 + self.post_layernorm = nn.LayerNorm(config.hidden_size,
263 + eps=config.layer_norm_eps)
264 +
265 + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
266 + hidden_states = self.embeddings(pixel_values)
267 + hidden_states = self.encoder(inputs_embeds=hidden_states)
268 +
269 + return self.post_layernorm(hidden_states)
vllm/model_executor/models/blip2.py new
+669
@@ -0,0 +1,669 @@
1 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
2 +
3 +import torch
4 +import torch.nn as nn
5 +from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
6 + apply_chunking_to_forward)
7 +
8 +from vllm.attention import AttentionMetadata
9 +from vllm.config import CacheConfig, MultiModalConfig
10 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11 +from vllm.model_executor.layers.activation import get_act_fn
12 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
13 +from vllm.model_executor.layers.quantization import QuantizationConfig
14 +from vllm.model_executor.layers.sampler import Sampler
15 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 +from vllm.model_executor.models.opt import OPTModel
17 +from vllm.model_executor.sampling_metadata import SamplingMetadata
18 +from vllm.multimodal import MULTIMODAL_REGISTRY
19 +from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
20 +
21 +from .blip import (BlipVisionModel, dummy_image_for_blip,
22 + get_max_blip_image_tokens)
23 +from .interfaces import SupportsVision
24 +from .utils import merge_vision_embeddings
25 +
26 +_KEYS_TO_MODIFY_MAPPING = {
27 + "language_model.lm_head": "lm_head",
28 + "language_model.model": "language_model",
29 +}
30 +
31 +
32 +class Blip2QFormerMultiHeadAttention(nn.Module):
33 +
34 + def __init__(
35 + self,
36 + config: Blip2QFormerConfig,
37 + *,
38 + quant_config: Optional[QuantizationConfig],
39 + cache_config: Optional[CacheConfig],
40 + is_cross_attention: bool = False,
41 + ) -> None:
42 + super().__init__()
43 +
44 + self.config = config
45 +
46 + if config.hidden_size % config.num_attention_heads != 0:
47 + raise ValueError(
48 + f"The hidden size ({config.hidden_size}) is not a multiple of "
49 + f"the number of attention heads ({config.num_attention_heads})"
50 + )
51 +
52 + self.num_attention_heads = config.num_attention_heads
53 + self.attention_head_size = (config.hidden_size //
54 + config.num_attention_heads)
55 + self.all_head_size = self.num_attention_heads * self.attention_head_size
56 + self.scaling = self.attention_head_size**-0.5
57 +
58 + self.query = nn.Linear(config.hidden_size, self.all_head_size)
59 + if is_cross_attention:
60 + kv_hidden_size = config.encoder_hidden_size
61 + else:
62 + kv_hidden_size = config.hidden_size
63 + self.key = nn.Linear(kv_hidden_size, self.all_head_size)
64 + self.value = nn.Linear(kv_hidden_size, self.all_head_size)
65 +
66 + self.position_embedding_type = getattr(config,
67 + "position_embedding_type",
68 + "absolute")
69 + if self.position_embedding_type != "absolute":
70 + raise NotImplementedError("Unsupported position_embedding_type: "
71 + f"{self.position_embedding_type}")
72 +
73 + self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
74 +
75 + def transpose_for_scores(self, x):
76 + x = x.view(*x.size()[:-1], self.num_attention_heads,
77 + self.attention_head_size)
78 + return x.permute(0, 2, 1, 3)
79 +
80 + def forward(
81 + self,
82 + hidden_states: torch.Tensor,
83 + encoder_hidden_states: Optional[torch.FloatTensor] = None,
84 + ):
85 + is_cross_attention = encoder_hidden_states is not None
86 +
87 + if is_cross_attention:
88 + key_layer = self.transpose_for_scores(
89 + self.key(encoder_hidden_states))
90 + value_layer = self.transpose_for_scores(
91 + self.value(encoder_hidden_states))
92 + else:
93 + key_layer = self.transpose_for_scores(self.key(hidden_states))
94 + value_layer = self.transpose_for_scores(self.value(hidden_states))
95 +
96 + mixed_query_layer = self.query(hidden_states)
97 +
98 + query_layer = self.transpose_for_scores(mixed_query_layer)
99 +
100 + attention_scores = torch.matmul(query_layer,
101 + key_layer.transpose(-1, -2))
102 + attention_probs = torch.softmax(attention_scores * self.scaling,
103 + dim=-1)
104 +
105 + # This is actually dropping out entire tokens to attend to, which might
106 + # seem a bit unusual, but is taken from the original Transformer paper.
107 + attention_probs_dropped = self.dropout(attention_probs)
108 +
109 + context_layer = torch.matmul(attention_probs_dropped, value_layer)
110 +
111 + context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
112 + context_layer = context_layer.view(*context_layer.size()[:-2],
113 + self.all_head_size)
114 +
115 + return context_layer
116 +
117 +
118 +class Blip2QFormerSelfOutput(nn.Module):
119 +
120 + def __init__(self, config: Blip2QFormerConfig) -> None:
121 + super().__init__()
122 +
123 + self.dense = nn.Linear(config.hidden_size, config.hidden_size)
124 + self.LayerNorm = nn.LayerNorm(config.hidden_size,
125 + eps=config.layer_norm_eps)
126 + self.dropout = nn.Dropout(config.hidden_dropout_prob)
127 +
128 + def forward(
129 + self,
130 + hidden_states: torch.Tensor,
131 + input_tensor: torch.Tensor,
132 + ) -> torch.Tensor:
133 + hidden_states = self.dense(hidden_states)
134 + hidden_states = self.dropout(hidden_states)
135 + hidden_states = self.LayerNorm(hidden_states + input_tensor)
136 + return hidden_states
137 +
138 +
139 +class Blip2QFormerAttention(nn.Module):
140 +
141 + def __init__(
142 + self,
143 + config: Blip2QFormerConfig,
144 + *,
145 + quant_config: Optional[QuantizationConfig],
146 + cache_config: Optional[CacheConfig],
147 + is_cross_attention: bool = False,
148 + ) -> None:
149 + super().__init__()
150 +
151 + self.attention = Blip2QFormerMultiHeadAttention(
152 + config,
153 + quant_config=quant_config,
154 + cache_config=cache_config,
155 + is_cross_attention=is_cross_attention,
156 + )
157 +
158 + self.output = Blip2QFormerSelfOutput(config)
159 +
160 + def forward(
161 + self,
162 + hidden_states: torch.Tensor,
163 + encoder_hidden_states: Optional[torch.FloatTensor] = None,
164 + ) -> Tuple[torch.Tensor]:
165 + self_output = self.attention(
166 + hidden_states,
167 + encoder_hidden_states=encoder_hidden_states,
168 + )
169 + attention_output = self.output(self_output, hidden_states)
170 +
171 + return attention_output
172 +
173 +
174 +class Blip2QFormerIntermediate(nn.Module):
175 +
176 + def __init__(self, config: Blip2QFormerConfig) -> None:
177 + super().__init__()
178 +
179 + self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
180 + self.intermediate_act_fn = get_act_fn(config.hidden_act)
181 +
182 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
183 + hidden_states = self.dense(hidden_states)
184 + hidden_states = self.intermediate_act_fn(hidden_states)
185 + return hidden_states
186 +
187 +
188 +class Blip2QFormerOutput(nn.Module):
189 +
190 + def __init__(self, config: Blip2QFormerConfig) -> None:
191 + super().__init__()
192 +
193 + self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
194 + self.LayerNorm = nn.LayerNorm(config.hidden_size,
195 + eps=config.layer_norm_eps)
196 + self.dropout = nn.Dropout(config.hidden_dropout_prob)
197 +
198 + def forward(
199 + self,
200 + hidden_states: torch.Tensor,
201 + input_tensor: torch.Tensor,
202 + ) -> torch.Tensor:
203 + hidden_states = self.dense(hidden_states)
204 + hidden_states = self.dropout(hidden_states)
205 + hidden_states = self.LayerNorm(hidden_states + input_tensor)
206 + return hidden_states
207 +
208 +
209 +class Blip2QFormerLayer(nn.Module):
210 +
211 + def __init__(
212 + self,
213 + config: Blip2QFormerConfig,
214 + *,
215 + quant_config: Optional[QuantizationConfig],
216 + cache_config: Optional[CacheConfig],
217 + layer_idx: int,
218 + ) -> None:
219 + super().__init__()
220 +
221 + self.chunk_size_feed_forward = config.chunk_size_feed_forward
222 + self.seq_len_dim = 1
223 + self.attention = Blip2QFormerAttention(config,
224 + quant_config=quant_config,
225 + cache_config=cache_config)
226 +
227 + self.layer_idx = layer_idx
228 +
229 + if layer_idx % config.cross_attention_frequency == 0:
230 + self.crossattention = Blip2QFormerAttention(
231 + config,
232 + quant_config=quant_config,
233 + cache_config=cache_config,
234 + is_cross_attention=True)
235 + self.has_cross_attention = True
236 + else:
237 + self.has_cross_attention = False
238 +
239 + self.intermediate_query = Blip2QFormerIntermediate(config)
240 + self.output_query = Blip2QFormerOutput(config)
241 +
242 + def forward(
243 + self,
244 + hidden_states: torch.FloatTensor,
245 + encoder_hidden_states: torch.FloatTensor,
246 + query_length: int,
247 + ):
248 + attention_output = self.attention(hidden_states)
249 +
250 + if query_length > 0:
251 + query_attention_output = attention_output[:, :query_length, :]
252 +
253 + if self.has_cross_attention:
254 + query_attention_output = self.crossattention(
255 + query_attention_output,
256 + encoder_hidden_states=encoder_hidden_states,
257 + )
258 +
259 + layer_output = apply_chunking_to_forward(
260 + self.feed_forward_chunk_query,
261 + self.chunk_size_feed_forward,
262 + self.seq_len_dim,
263 + query_attention_output,
264 + )
265 +
266 + if attention_output.shape[1] > query_length:
267 + layer_output_text = apply_chunking_to_forward(
268 + self.feed_forward_chunk,
269 + self.chunk_size_feed_forward,
270 + self.seq_len_dim,
271 + attention_output[:, query_length:, :],
272 + )
273 + layer_output = torch.cat([layer_output, layer_output_text],
274 + dim=1)
275 + else:
276 + layer_output = apply_chunking_to_forward(
277 + self.feed_forward_chunk,
278 + self.chunk_size_feed_forward,
279 + self.seq_len_dim,
280 + attention_output,
281 + )
282 +
283 + return layer_output
284 +
285 + def feed_forward_chunk(self,
286 + attention_output: torch.Tensor) -> torch.Tensor:
287 + intermediate_output = self.intermediate(attention_output)
288 + layer_output = self.output(intermediate_output, attention_output)
289 + return layer_output
290 +
291 + def feed_forward_chunk_query(
292 + self, attention_output: torch.Tensor) -> torch.Tensor:
293 + intermediate_output = self.intermediate_query(attention_output)
294 + layer_output = self.output_query(intermediate_output, attention_output)
295 + return layer_output
296 +
297 +
298 +class Blip2QFormerEncoder(nn.Module):
299 +
300 + def __init__(
301 + self,
302 + config: Blip2QFormerConfig,
303 + *,
304 + quant_config: Optional[QuantizationConfig],
305 + cache_config: Optional[CacheConfig],
306 + ) -> None:
307 + super().__init__()
308 +
309 + self.config = config
310 +
311 + self.layer = nn.ModuleList([
312 + Blip2QFormerLayer(config,
313 + quant_config=quant_config,
314 + cache_config=cache_config,
315 + layer_idx=layer_idx)
316 + for layer_idx in range(config.num_hidden_layers)
317 + ])
318 +
319 + def forward(
320 + self,
321 + hidden_states: torch.FloatTensor,
322 + encoder_hidden_states: torch.FloatTensor,
323 + query_length: int,
324 + ) -> torch.Tensor:
325 + for i in range(self.config.num_hidden_layers):
326 + layer_module = self.layer[i]
327 +
328 + hidden_states = layer_module(
329 + hidden_states,
330 + encoder_hidden_states=encoder_hidden_states,
331 + query_length=query_length,
332 + )
333 +
334 + return hidden_states
335 +
336 +
337 +# Adapted from https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1025
338 +class Blip2QFormerModel(nn.Module):
339 +
340 + def __init__(
341 + self,
342 + config: Blip2QFormerConfig,
343 + *,
344 + quant_config: Optional[QuantizationConfig],
345 + cache_config: Optional[CacheConfig],
346 + ) -> None:
347 + super().__init__()
348 +
349 + self.config = config
350 +
351 + self.layernorm = nn.LayerNorm(config.hidden_size,
352 + eps=config.layer_norm_eps)
353 + self.dropout = nn.Dropout(config.hidden_dropout_prob)
354 +
355 + self.encoder = Blip2QFormerEncoder(config,
356 + quant_config=quant_config,
357 + cache_config=cache_config)
358 +
359 + def forward(
360 + self,
361 + query_embeds: torch.FloatTensor,
362 + encoder_hidden_states: torch.FloatTensor,
363 + ) -> torch.Tensor:
364 + query_length = query_embeds.shape[1]
365 +
366 + embedding_output = self.layernorm(query_embeds)
367 + embedding_output = self.dropout(embedding_output)
368 +
369 + sequence_output = self.encoder(
370 + embedding_output,
371 + encoder_hidden_states=encoder_hidden_states,
372 + query_length=query_length,
373 + )
374 +
375 + return sequence_output
376 +
377 +
378 +class Blip2ImagePixelInputs(TypedDict):
379 + type: Literal["pixel_values"]
380 + data: torch.Tensor
381 + """Shape: (batch_size, num_channels, height, width)"""
382 +
383 +
384 +Blip2ImageInputs = Blip2ImagePixelInputs
385 +
386 +# We use this internally as placeholders since there is no image token
387 +# defined on the HuggingFace repo
388 +BLIP2_IMAGE_TOKEN = "<image>"
389 +BLIP2_IMAGE_TOKEN_ID = 50265
390 +
391 +
392 +def get_blip2_image_feature_size(hf_config: Blip2Config) -> int:
393 + return hf_config.num_query_tokens
394 +
395 +
396 +def get_max_blip2_image_tokens(ctx: InputContext):
397 + hf_config = ctx.get_hf_config(Blip2Config)
398 + vision_config = hf_config.vision_config
399 +
400 + if isinstance(vision_config, Blip2VisionConfig):
401 + return get_max_blip_image_tokens(vision_config)
402 +
403 + msg = f"Unsupported vision config: {type(vision_config)}"
404 + raise NotImplementedError(msg)
405 +
406 +
407 +def dummy_data_for_blip2(ctx: InputContext, seq_len: int):
408 + hf_config = ctx.get_hf_config(Blip2Config)
409 + vision_config = hf_config.vision_config
410 +
411 + image_feature_size = get_blip2_image_feature_size(hf_config)
412 + token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
413 + token_ids += [0] * (seq_len - image_feature_size)
414 + seq_data = SequenceData(token_ids)
415 +
416 + if isinstance(vision_config, Blip2VisionConfig):
417 + mm_data = dummy_image_for_blip(vision_config)
418 +
419 + return seq_data, mm_data
420 +
421 + msg = f"Unsupported vision config: {type(vision_config)}"
422 + raise NotImplementedError(msg)
423 +
424 +
425 +def input_processor_for_blip2(ctx: InputContext, llm_inputs: LLMInputs):
426 + multi_modal_data = llm_inputs.get("multi_modal_data")
427 + if multi_modal_data is None or "image" not in multi_modal_data:
428 + return llm_inputs
429 +
430 + hf_config = ctx.get_hf_config(Blip2Config)
431 + image_feature_size = get_blip2_image_feature_size(hf_config)
432 +
433 + # The original model places image tokens at the front
434 + # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1514
435 + new_token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
436 + new_token_ids += llm_inputs["prompt_token_ids"]
437 +
438 + new_prompt = llm_inputs.get("prompt")
439 + if new_prompt is not None:
440 + new_prompt = BLIP2_IMAGE_TOKEN * image_feature_size + new_prompt
441 +
442 + return LLMInputs(prompt_token_ids=new_token_ids,
443 + prompt=new_prompt,
444 + multi_modal_data=multi_modal_data)
445 +
446 +
447 +@MULTIMODAL_REGISTRY.register_image_input_mapper()
448 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_blip2_image_tokens)
449 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_blip2)
450 +@INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
451 +class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
452 +
453 + def __init__(self,
454 + config: Blip2Config,
455 + multimodal_config: MultiModalConfig,
456 + cache_config: Optional[CacheConfig] = None,
457 + quant_config: Optional[QuantizationConfig] = None) -> None:
458 +
459 + super().__init__()
460 +
461 + self.config = config
462 + self.multimodal_config = multimodal_config
463 +
464 + # TODO: Optionally initializes this for supporting embeddings.
465 + self.vision_model = BlipVisionModel(config.vision_config)
466 +
467 + self.query_tokens = nn.Parameter(
468 + torch.zeros(1, config.num_query_tokens,
469 + config.qformer_config.hidden_size))
470 +
471 + self.qformer = Blip2QFormerModel(config.qformer_config,
472 + cache_config=cache_config,
473 + quant_config=quant_config)
474 +
475 + self.language_projection = nn.Linear(
476 + config.qformer_config.hidden_size,
477 + config.text_config.hidden_size,
478 + bias=True,
479 + )
480 +
481 + self.quant_config = quant_config
482 +
483 + self.language_model = OPTModel(config.text_config, cache_config,
484 + quant_config)
485 +
486 + self.unpadded_vocab_size = config.text_config.vocab_size
487 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size)
488 + self.sampler = Sampler()
489 +
490 + def get_lm_head(self):
491 + return self.language_model.decoder.embed_tokens
492 +
493 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
494 + h = w = self.config.vision_config.image_size
495 + expected_dims = (3, h, w)
496 + actual_dims = tuple(data.shape[1:])
497 +
498 + if actual_dims != expected_dims:
499 + expected_expr = ("batch_size", *map(str, expected_dims))
500 + raise ValueError(
501 + f"The expected shape of pixel values is {expected_expr}. "
502 + f"You supplied {tuple(data.shape)}.")
503 +
504 + return data
505 +
506 + def _parse_and_validate_image_input(
507 + self, **kwargs: object) -> Optional[Blip2ImageInputs]:
508 + pixel_values = kwargs.pop("pixel_values", None)
509 +
510 + if pixel_values is None:
511 + return None
512 +
513 + if not isinstance(pixel_values, torch.Tensor):
514 + raise ValueError("Incorrect type of pixel values. "
515 + f"Got type: {type(pixel_values)}")
516 +
517 + return Blip2ImagePixelInputs(
518 + type="pixel_values",
519 + data=self._validate_pixel_values(pixel_values),
520 + )
521 +
522 + def _image_pixels_to_features(self, vision_model: BlipVisionModel,
523 + pixel_values: torch.Tensor) -> torch.Tensor:
524 +
525 + # NOTE: we skip the step to select the vision feature layer since
526 + # this is already done inside the vision tower
527 + image_features = vision_model(pixel_values)
528 +
529 + return image_features
530 +
531 + def _process_image_pixels(self,
532 + inputs: Blip2ImagePixelInputs) -> torch.Tensor:
533 + assert self.vision_model is not None
534 +
535 + pixel_values = inputs["data"]
536 +
537 + return self._image_pixels_to_features(self.vision_model, pixel_values)
538 +
539 + def _process_image_input(self,
540 + image_input: Blip2ImageInputs) -> torch.Tensor:
541 + assert self.vision_model is not None
542 + image_features = self._process_image_pixels(image_input)
543 +
544 + query_tokens = self.query_tokens.expand(image_features.shape[0], -1,
545 + -1)
546 + query_output = self.qformer(
547 + query_embeds=query_tokens,
548 + encoder_hidden_states=image_features,
549 + )
550 +
551 + return self.language_projection(query_output)
552 +
553 + def forward(
554 + self,
555 + input_ids: torch.Tensor,
556 + positions: torch.Tensor,
557 + kv_caches: List[torch.Tensor],
558 + attn_metadata: AttentionMetadata,
559 + intermediate_tensors: Optional[IntermediateTensors] = None,
560 + **kwargs: object,
561 + ) -> SamplerOutput:
562 + """Run forward pass for BLIP-2.
563 +
564 + One key thing to understand is the `input_ids` already accounts for the
565 + positions of the to-be-inserted image embeddings.
566 +
567 + Concretely, consider a text prompt:
568 + `"Question: What's the content of the image? Answer:"`.
569 +
570 + Tokenizer outputs:
571 + `[2, 45641, 35, 653, 18, 5, 1383, 9, 5, 2274, 116, 31652, 35]`.
572 +
573 + To reserve space in KV cache, we have to insert placeholder tokens
574 + before they are inputted to the model, so the input processor prepends
575 + dummy tokens (denoted as `50265`), resulting in:
576 + `[50265, ..., 50265, 2, 45641, 35, ..., 31652, 35]`.
577 +
578 + We insert 32 tokens since it corresponds to the number of query
579 + embeddings outputted by the Q-Former and inputted to the language model.
580 +
581 + This way, the `positions` and `attn_metadata` are consistent
582 + with the `input_ids`.
583 +
584 + Args:
585 + input_ids: Flattened (concatenated) input_ids corresponding to a
586 + batch.
587 + pixel_values: The pixels in each input image.
588 +
589 + See also:
590 + :class:`Blip2ImageInputs`
591 + """
592 + image_input = self._parse_and_validate_image_input(**kwargs)
593 +
594 + if image_input is not None:
595 + vision_embeddings = self._process_image_input(image_input)
596 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
597 +
598 + inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
599 + vision_embeddings,
600 + BLIP2_IMAGE_TOKEN_ID)
601 +
602 + input_ids = None
603 + else:
604 + inputs_embeds = None
605 +
606 + hidden_states = self.language_model(input_ids,
607 + positions,
608 + kv_caches,
609 + attn_metadata,
610 + inputs_embeds=inputs_embeds)
611 +
612 + return hidden_states
613 +
614 + def compute_logits(self, hidden_states: torch.Tensor,
615 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
616 + logits = self.logits_processor(self.get_lm_head(), hidden_states,
617 + sampling_metadata)
618 + return logits
619 +
620 + def sample(
621 + self,
622 + logits: torch.Tensor,
623 + sampling_metadata: SamplingMetadata,
624 + ) -> Optional[SamplerOutput]:
625 + next_tokens = self.sampler(logits, sampling_metadata)
626 + return next_tokens
627 +
628 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
629 + # only doing this for language model part for now.
630 + stacked_params_mapping = [
631 + # (param_name, shard_name, shard_id)
632 + ("qkv_proj", "q_proj", "q"),
633 + ("qkv_proj", "k_proj", "k"),
634 + ("qkv_proj", "v_proj", "v"),
635 + ("gate_up_proj", "gate_proj", 0),
636 + ("gate_up_proj", "up_proj", 1),
637 + ]
638 + params_dict = dict(self.named_parameters())
639 +
640 + for name, loaded_weight in weights:
641 + if "lm_head.weight" in name:
642 + continue
643 + if "rotary_emb.inv_freq" in name:
644 + continue
645 + for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
646 + if key_to_modify in name:
647 + name = name.replace(key_to_modify, new_key)
648 + use_default_weight_loading = False
649 + if "vision" in name:
650 + if self.vision_model is not None:
651 + # We only do sharding for language model and
652 + # not vision model for now.
653 + use_default_weight_loading = True
654 + else:
655 + for (param_name, weight_name,
656 + shard_id) in stacked_params_mapping:
657 + if weight_name not in name:
658 + continue
659 + param = params_dict[name.replace(weight_name, param_name)]
660 + weight_loader = param.weight_loader
661 + weight_loader(param, loaded_weight, shard_id)
662 + break
663 + else:
664 + use_default_weight_loading = True
665 + if use_default_weight_loading:
666 + param = params_dict[name]
667 + weight_loader = getattr(param, "weight_loader",
668 + default_weight_loader)
669 + weight_loader(param, loaded_weight)
vllm/model_executor/models/opt.py
+15 -2
@@ -237,14 +237,19 @@ class OPTDecoder(nn.Module):
237 for _ in range(config.num_hidden_layers)
238 ])
239
240 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
241 + return self.embed_tokens(input_ids)
242 +
243 def forward(
244 self,
245 input_ids: torch.Tensor,
246 positions: torch.Tensor,
247 kv_caches: List[torch.Tensor],
248 attn_metadata: AttentionMetadata,
249 + inputs_embeds: Optional[torch.Tensor] = None,
250 ) -> torch.Tensor:
247 - inputs_embeds = self.embed_tokens(input_ids)
251 + if inputs_embeds is None:
252 + inputs_embeds = self.get_input_embeddings(input_ids)
253 pos_embeds = self.embed_positions(positions)
254 if self.project_in is not None:
255 inputs_embeds, _ = self.project_in(inputs_embeds)
@@ -272,14 +277,22 @@ class OPTModel(nn.Module):
277 super().__init__()
278 self.decoder = OPTDecoder(config, cache_config, quant_config)
279
280 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
281 + return self.decoder.get_input_embeddings(input_ids)
282 +
283 def forward(
284 self,
285 input_ids: torch.Tensor,
286 positions: torch.Tensor,
287 kv_caches: List[torch.Tensor],
288 attn_metadata: AttentionMetadata,
289 + inputs_embeds: Optional[torch.Tensor] = None,
290 ) -> torch.Tensor:
282 - return self.decoder(input_ids, positions, kv_caches, attn_metadata)
291 + return self.decoder(input_ids,
292 + positions,
293 + kv_caches,
294 + attn_metadata,
295 + inputs_embeds=inputs_embeds)
296
297
298 class OPTForCausalLM(nn.Module):
vllm/multimodal/base.py
+6 -5
@@ -1,8 +1,9 @@
1 import sys
2 from abc import ABC, abstractmethod
3 from collections import UserDict, defaultdict
4 -from typing import (Any, Callable, Dict, List, Optional, Type, TypedDict,
5 - TypeVar, Union, cast)
4 +from typing import Any, Callable, Dict, List, Optional
5 +from typing import Sequence as GenericSequence
6 +from typing import Type, TypedDict, TypeVar, Union, cast
7
8 import torch
9 import torch.types
@@ -15,13 +16,13 @@ from vllm.logger import init_logger
16
17 logger = init_logger(__name__)
18
18 -NestedTensors = Union[List[torch.Tensor], torch.Tensor]
19 +NestedTensors = Union[GenericSequence[torch.Tensor], torch.Tensor]
20 """
21 Use a list instead of a tensor if the dimensions of each element do not match.
22 Currently only supports up to singly nested list of tensors.
23 """
24
24 -BatchedTensors = Union[List[NestedTensors], NestedTensors]
25 +BatchedTensors = Union[GenericSequence[NestedTensors], NestedTensors]
26 """
27 If each input tensor in the batch has the same size, this is a single batched
28 tensor; otherwise, this is a list of :class:`NestedTensors` with one element
@@ -53,7 +54,7 @@ class MultiModalInputs(_MultiModalInputsBase):
54 # may be list rather than tensors
55 if isinstance(tensors[0], list):
56 return [[t.to(device=device) for t in tensor[0]]
56 - for tensor in tensors]
57 + for tensor in cast(List[List[torch.Tensor]], tensors)]
58
59 tensors_ = cast(List[torch.Tensor], tensors)
60