@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 179a6a36f2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model]Refactor MiniCPMV (#7020)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Jee Jee Li committed Aug 4, 2024 at 16:12 UTC 179a6a36f2a585df49ce9c26701b1b9d894bd00e
4 files changed +937 -386
docs/source/models/supported_models.rst
+1 -1
@@ -220,7 +220,7 @@ Vision Language Models
220 - Phi-3-Vision
221 - :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
222 -
223 - * - :code:`MiniCPM-V`
223 + * - :code:`MiniCPMV`
224 - MiniCPM-V
225 - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
226 -
vllm/model_executor/models/idefics2_vision_model.py new
+296
@@ -0,0 +1,296 @@
1 +# coding=utf-8
2 +
3 +# adapted from https://github.com/huggingface/transformers/blob/v4.43.2/src/transformers/models/idefics2/modeling_idefics2.py
4 +# Copyright 2024 The vLLM team.
5 +# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# Licensed under the Apache License, Version 2.0 (the "License");
8 +# you may not use this file except in compliance with the License.
9 +# You may obtain a copy of the License at
10 +#
11 +# http://www.apache.org/licenses/LICENSE-2.0
12 +#
13 +# Unless required by applicable law or agreed to in writing, software
14 +# distributed under the License is distributed on an "AS IS" BASIS,
15 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 +# See the License for the specific language governing permissions and
17 +# limitations under the License.
18 +"""PyTorch Idefics2 model."""
19 +
20 +from typing import Optional
21 +
22 +import torch
23 +from torch import nn
24 +from transformers.models.idefics2.configuration_idefics2 import (
25 + Idefics2Config, Idefics2VisionConfig)
26 +from xformers import ops as xops
27 +
28 +from vllm.distributed import divide, get_tensor_model_parallel_world_size
29 +from vllm.model_executor.layers.activation import get_act_fn
30 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 + QKVParallelLinear,
32 + RowParallelLinear)
33 +from vllm.model_executor.layers.quantization import QuantizationConfig
34 +
35 +
36 +class Idefics2VisionEmbeddings(nn.Module):
37 + """
38 + This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings
39 + ` to enable images of variable
40 + resolution.
41 +
42 + The modifications are adapted from [Patch n' Pack: NaViT, a Vision
43 + Transformer for any Aspect Ratio and Resolution](https://arxiv.org/abs/2307.06304)
44 + which allows treating images in their native aspect ratio and without the
45 + need to resize them to the same fixed size. In particular, we start from the
46 + original pre-trained SigLIP model(which uses images of fixed-size square
47 + images) and adapt it by training on images of variable resolutions.
48 + """
49 +
50 + def __init__(self, config: Idefics2VisionConfig):
51 + super().__init__()
52 + self.embed_dim = config.hidden_size
53 + self.image_size = config.image_size
54 + self.patch_size = config.patch_size
55 + self.patch_embedding = nn.Conv2d(
56 + in_channels=config.num_channels,
57 + out_channels=self.embed_dim,
58 + kernel_size=self.patch_size,
59 + stride=self.patch_size,
60 + padding="valid",
61 + )
62 + self.num_patches_per_side = self.image_size // self.patch_size
63 + self.num_patches = self.num_patches_per_side**2
64 + self.num_positions = self.num_patches
65 + self.position_embedding = nn.Embedding(self.num_positions,
66 + self.embed_dim)
67 +
68 + def forward(
69 + self,
70 + pixel_values: torch.FloatTensor,
71 + patch_attention_mask: torch.BoolTensor,
72 + ) -> torch.Tensor:
73 + batch_size, _, max_im_h, max_im_w = pixel_values.shape
74 + patch_embeds = self.patch_embedding(pixel_values)
75 + embeddings = patch_embeds.flatten(2).transpose(1, 2)
76 + max_nb_patches_h, max_nb_patches_w = (
77 + max_im_h // self.patch_size,
78 + max_im_w // self.patch_size,
79 + )
80 + boundaries = torch.arange(1 / self.num_patches_per_side, 1.0,
81 + 1 / self.num_patches_per_side)
82 + position_ids = torch.full(size=(batch_size,
83 + max_nb_patches_h * max_nb_patches_w),
84 + fill_value=0)
85 +
86 + for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
87 + nb_patches_h = p_attn_mask[:, 0].sum()
88 + nb_patches_w = p_attn_mask[0].sum()
89 + fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
90 + fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
91 + bucket_coords_h = torch.bucketize(fractional_coords_h,
92 + boundaries,
93 + right=True)
94 + bucket_coords_w = torch.bucketize(fractional_coords_w,
95 + boundaries,
96 + right=True)
97 + pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side +
98 + bucket_coords_w).flatten()
99 + position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
100 + position_ids = position_ids.to(self.position_embedding.weight.device)
101 + embeddings = embeddings + self.position_embedding(position_ids)
102 + return embeddings
103 +
104 +
105 +class Idefics2VisionAttention(nn.Module):
106 + """Multi-headed attention from 'Attention Is All You Need' paper"""
107 +
108 + def __init__(
109 + self,
110 + config: Idefics2Config,
111 + quant_config: Optional[QuantizationConfig] = None,
112 + ):
113 + super().__init__()
114 + self.config = config
115 + self.embed_dim = config.hidden_size
116 + self.num_heads = config.num_attention_heads
117 + self.head_dim = self.embed_dim // self.num_heads
118 + if self.head_dim * self.num_heads != self.embed_dim:
119 + raise ValueError(
120 + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" # noqa: E501
121 + f" {self.num_heads}).")
122 + self.scale = self.head_dim**-0.5
123 + self.dropout = config.attention_dropout
124 + self.qkv_proj = QKVParallelLinear(
125 + self.embed_dim,
126 + self.head_dim,
127 + self.num_heads,
128 + quant_config=quant_config,
129 + )
130 + self.out_proj = RowParallelLinear(
131 + self.embed_dim,
132 + self.embed_dim,
133 + bias=True,
134 + quant_config=quant_config,
135 + )
136 + self.tp_size = get_tensor_model_parallel_world_size()
137 + self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
138 + self.is_causal = False
139 +
140 + def forward(
141 + self,
142 + hidden_states: torch.Tensor,
143 + ) -> torch.Tensor:
144 + batch_size, q_len, _ = hidden_states.size()
145 + qkv, _ = self.qkv_proj(
146 + hidden_states
147 + ) # batch_size, q_len, 3 * num_heads_per_partition * head_dim
148 + query_states, key_states, value_states = qkv.chunk(3, dim=-1)
149 + query_states = query_states.view(batch_size, q_len,
150 + self.num_heads_per_partition,
151 + self.head_dim)
152 + key_states = key_states.view(batch_size, q_len,
153 + self.num_heads_per_partition,
154 + self.head_dim)
155 + value_states = value_states.view(batch_size, q_len,
156 + self.num_heads_per_partition,
157 + self.head_dim)
158 + # see: https://facebookresearch.github.io/xformers/components/ops.html
159 + out = xops.memory_efficient_attention_forward(
160 + query_states,
161 + key_states,
162 + value_states,
163 + p=self.dropout,
164 + scale=self.scale,
165 + )
166 + out = out.view(batch_size, q_len, -1)
167 + attn_output, _ = self.out_proj(out)
168 + return attn_output
169 +
170 +
171 +class Idefics2VisionMLP(nn.Module):
172 +
173 + def __init__(
174 + self,
175 + config: Idefics2Config,
176 + quant_config: Optional[QuantizationConfig] = None,
177 + ):
178 + super().__init__()
179 + self.config = config
180 + self.activation_fn = get_act_fn(config.hidden_act)
181 + self.fc1 = ColumnParallelLinear(
182 + config.hidden_size,
183 + config.intermediate_size,
184 + bias=True,
185 + quant_config=quant_config,
186 + )
187 + self.fc2 = RowParallelLinear(
188 + config.intermediate_size,
189 + config.hidden_size,
190 + bias=True,
191 + quant_config=quant_config,
192 + )
193 +
194 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
195 + hidden_states, _ = self.fc1(hidden_states)
196 + hidden_states = self.activation_fn(hidden_states)
197 + hidden_states, _ = self.fc2(hidden_states)
198 + return hidden_states
199 +
200 +
201 +class Idefics2EncoderLayer(nn.Module):
202 +
203 + def __init__(self, config: Idefics2Config):
204 + super().__init__()
205 + self.embed_dim = config.hidden_size
206 + self.self_attn = Idefics2VisionAttention(config)
207 + self.layer_norm1 = nn.LayerNorm(self.embed_dim,
208 + eps=config.layer_norm_eps)
209 + self.mlp = Idefics2VisionMLP(config)
210 + self.layer_norm2 = nn.LayerNorm(self.embed_dim,
211 + eps=config.layer_norm_eps)
212 +
213 + def forward(
214 + self,
215 + hidden_states: torch.Tensor,
216 + ) -> torch.Tensor:
217 + """
218 + Args:
219 + hidden_states (`torch.FloatTensor`):
220 + Input to the layer of shape `(batch, seq_len, embed_dim)`.
221 +
222 + """
223 + residual = hidden_states
224 + hidden_states = self.layer_norm1(hidden_states)
225 + hidden_states = self.self_attn(hidden_states)
226 + hidden_states = residual + hidden_states
227 + residual = hidden_states
228 + hidden_states = self.layer_norm2(hidden_states)
229 + hidden_states = self.mlp(hidden_states)
230 + hidden_states = residual + hidden_states
231 + return hidden_states
232 +
233 +
234 +class Idefics2Encoder(nn.Module):
235 + """
236 + Transformer encoder consisting of `config.num_hidden_layers` self attention
237 + layers. Each layer is a
238 + [`Idefics2EncoderLayer`].
239 +
240 + Args:
241 + config: Idefics2Config
242 + """
243 +
244 + def __init__(self, config: Idefics2Config):
245 + super().__init__()
246 + self.config = config
247 + self.layers = nn.ModuleList([
248 + Idefics2EncoderLayer(config)
249 + for _ in range(config.num_hidden_layers)
250 + ])
251 +
252 + def forward(
253 + self,
254 + inputs_embeds: torch.Tensor,
255 + ) -> torch.Tensor:
256 + r"""
257 + Args:
258 + inputs_embeds (torch.Tensor):
259 + Optionally, instead of passing `input_ids` you can choose to
260 + directly pass an embedded representation.
261 + This is useful if you want more control over how to convert
262 + `input_ids` indices into associated vectorsthan the model's
263 + internal embedding lookup matrix.
264 + """
265 + hidden_states = inputs_embeds
266 + for encoder_layer in self.layers:
267 + layer_outputs = encoder_layer(hidden_states)
268 + hidden_states = layer_outputs
269 + return hidden_states
270 +
271 +
272 +class Idefics2VisionTransformer(nn.Module):
273 +
274 + def __init__(self, config: Idefics2VisionConfig):
275 + super().__init__()
276 + embed_dim = config.hidden_size
277 + self.config = config
278 + self.embeddings = Idefics2VisionEmbeddings(config)
279 + self.encoder = Idefics2Encoder(config)
280 + self.post_layernorm = nn.LayerNorm(embed_dim,
281 + eps=config.layer_norm_eps)
282 +
283 + def get_input_embeddings(self):
284 + return self.embeddings
285 +
286 + def forward(
287 + self,
288 + pixel_values,
289 + patch_attention_mask: Optional[torch.BoolTensor] = None,
290 + ) -> torch.tensor:
291 + hidden_states = self.embeddings(
292 + pixel_values=pixel_values,
293 + patch_attention_mask=patch_attention_mask)
294 + encoder_outputs = self.encoder(hidden_states)
295 + last_hidden_state = self.post_layernorm(encoder_outputs)
296 + return last_hidden_state
vllm/model_executor/models/minicpmv.py
+639 -384
@@ -24,7 +24,8 @@
24 import math
25 import re
26 from functools import partial
27 -from typing import Dict, Iterable, List, Optional, Tuple, Union
27 +from typing import (Any, Callable, Iterable, List, Optional, Tuple, TypedDict,
28 + Union)
29
30 import numpy as np
31 import torch
@@ -38,11 +39,14 @@ from transformers.configuration_utils import PretrainedConfig
39 from vllm.attention import AttentionMetadata
40 from vllm.config import CacheConfig, MultiModalConfig
41 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
42 +from vllm.logger import init_logger
43 +from vllm.model_executor.layers.linear import ReplicatedLinear
44 from vllm.model_executor.layers.logits_processor import LogitsProcessor
45 from vllm.model_executor.layers.quantization.base_config import (
46 QuantizationConfig)
47 from vllm.model_executor.layers.sampler import Sampler
48 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
49 +from vllm.model_executor.model_loader.utils import set_default_torch_dtype
50 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51 from vllm.model_executor.models.interfaces import SupportsVision
52 from vllm.model_executor.models.llama import LlamaModel
@@ -54,12 +58,45 @@ from vllm.multimodal.image import (cached_get_image_processor,
58 cached_get_tokenizer)
59 from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
60
61 +from .idefics2_vision_model import Idefics2VisionTransformer
62 +
63 +logger = init_logger(__name__)
64 +
65 _KEYS_TO_MODIFY_MAPPING = {
66 "llm.lm_head": "lm_head",
67 "llm.model": "llm",
68 }
69
70
71 +class MiniCPMVImagePixelInputs(TypedDict):
72 + pixel_values: List[torch.Tensor]
73 + """
74 + Shape: `(batch_size * num_images, num_channels, height, width)`
75 +
76 + Note that the image size may vary, so we pass it as a list
77 + instead of a batched tensor.
78 + """
79 +
80 + image_bounds: torch.Tensor
81 + """
82 + Shape: `(batch_size * num_images, 2)`
83 +
84 + This should be in `(start, stop)` format.
85 + """
86 +
87 + tgt_sizes: torch.Tensor
88 + """
89 + Shape: `(batch_size * num_images, 2)`
90 +
91 + This should be in `(height, width)` format.
92 + """
93 +
94 +
95 +MiniCPMVImageInputs = MiniCPMVImagePixelInputs
96 +
97 +DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
98 +
99 +
100 def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor):
101 # abs_pos: L, C
102 # tgt_size: (H, W)
@@ -68,23 +105,25 @@ def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor):
105 # tgt_size = int(math.sqrt(tgt_size))
106 dtype = abs_pos.dtype
107
71 - return F.interpolate(
108 + return (F.interpolate(
109 abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
110 size=(tgt_size[0], tgt_size[1]),
111 mode="bicubic",
112 align_corners=False,
76 - ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)
113 + ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype))
114
115
116 # https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
80 -def get_2d_sincos_pos_embed(embed_dim: int,
81 - grid_size: Union[int, Tuple[int, int]],
82 - cls_token: bool = False,
83 - version: Tuple[int, int] = (2, 0)):
117 +def get_2d_sincos_pos_embed(
118 + embed_dim: int,
119 + grid_size: Union[int, Tuple[int, int]],
120 + cls_token: bool = False,
121 + version: Tuple[int, int] = (2, 0),
122 +):
123 """
124 grid_size: int of the grid height and width
125 return:
87 - pos_embed: [grid_size*grid_size, embed_dim] or
126 + pos_embed: [grid_size*grid_size, embed_dim] or
127 [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
128 """
129 if isinstance(grid_size, int):
@@ -109,7 +148,7 @@ def get_2d_sincos_pos_embed(embed_dim: int,
148
149
150 def get_2d_sincos_pos_embed_from_grid(embed_dim: int,
112 - grid: Union[int, Tuple[int, int]],
151 + grid: np.ndarray,
152 version: Tuple[int, int] = (2, 0)):
153 assert embed_dim % 2 == 0
154
@@ -127,7 +166,7 @@ def get_2d_sincos_pos_embed_from_grid(embed_dim: int,
166
167
168 def get_1d_sincos_pos_embed_from_grid(embed_dim: int,
130 - pos: int,
169 + pos: np.ndarray,
170 version: Tuple[int, int] = (2, 0)):
171 """
172 embed_dim: output dimension for each position
@@ -136,24 +175,24 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim: int,
175 """
176 assert embed_dim % 2 == 0
177 omega = np.arange(embed_dim // 2, dtype=np.float32)
139 - omega /= embed_dim / 2.
140 - omega = 1. / 10000**omega # (D/2,)
178 + omega /= embed_dim / 2.0
179 + omega = 1.0 / 10000**omega # (D/2,)
180
181 if version == (2, 0):
182 pos = pos.reshape(-1) # (M,)
144 - out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
183 + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
184 emb_sin = np.sin(out) # (M, D/2)
185 emb_cos = np.cos(out) # (M, D/2)
186 emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
187 else:
149 - out = np.einsum('hw,d->hwd', pos, omega) # (H, W, D/2), outer product
188 + out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
189 emb_sin = np.sin(out) # (H, W, D/2)
190 emb_cos = np.cos(out) # (H, W, D/2)
191 emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
192 return emb
193
194
156 -class Resampler(nn.Module):
195 +class BaseResampler(nn.Module):
196 """
197 A 2D perceiver-resampler network with one cross attention layers by
198 (grid_size**2) learnable queries and 2d sincos pos_emb
@@ -161,89 +200,151 @@ class Resampler(nn.Module):
200 A tensor with the shape of (grid_size**2, embed_dim)
201 """
202
164 - default_norm_layer = partial(nn.LayerNorm, eps=1e-6)
165 -
166 - def __init__(self,
167 - num_queries: int,
168 - grid_size: int,
169 - embed_dim: int,
170 - num_heads: int,
171 - kv_dim: Optional[int] = None,
172 - norm_layer: nn.Module = default_norm_layer,
173 - adaptive: bool = False,
174 - max_size: Tuple[int, int] = (70, 70),
175 - version: Tuple[int, int] = (2, 0)):
203 + def __init__(
204 + self,
205 + num_queries: int,
206 + embed_dim: int,
207 + num_heads: int,
208 + kv_dim: Optional[int] = None,
209 + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
210 + ) -> None:
211 super().__init__()
212
178 - self.version = version
179 - if self.version == (2, 0):
180 - self.num_queries = grid_size**2
181 - else:
182 - self.num_queries = num_queries
183 - self.max_size = max_size
213 + self.num_queries = num_queries
214 self.embed_dim = embed_dim
215 self.num_heads = num_heads
186 - self.adaptive = adaptive
216
217 self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
189 - trunc_normal_(self.query, std=.02)
218 + trunc_normal_(self.query, std=0.02)
219
220 if kv_dim is not None and kv_dim != embed_dim:
192 - self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
221 + self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
222 else:
194 - self.kv_proj = nn.Identity()
223 + # Maintain the same return value with ReplicatedLinear.forward
224 + self.kv_proj = lambda *args, **kwargs: (
225 + nn.Identity()(*args, **kwargs),
226 + None,
227 + )
228
229 self.attn = nn.MultiheadAttention(embed_dim, num_heads)
230 self.ln_q = norm_layer(embed_dim)
231 self.ln_kv = norm_layer(embed_dim)
199 -
232 self.ln_post = norm_layer(embed_dim)
233 self.proj = nn.Parameter(
234 (embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
235
204 - if self.version == (2, 0):
205 - self.pos_embed = nn.Parameter(
206 - torch.from_numpy(
207 - get_2d_sincos_pos_embed(
208 - embed_dim, grid_size,
209 - version=self.version)).float()).requires_grad_(False)
236 + def _init_weights(self, m: nn.Module) -> None:
237 + if isinstance(m, nn.Linear):
238 + trunc_normal_(m.weight, std=0.02)
239 + if isinstance(m, nn.Linear) and m.bias is not None:
240 + nn.init.constant_(m.bias, 0)
241 + elif isinstance(m, nn.LayerNorm):
242 + nn.init.constant_(m.bias, 0)
243 + nn.init.constant_(m.weight, 1.0)
244 +
245 + def _repeat(self, query, N: int):
246 + return query.unsqueeze(1).repeat(1, N, 1)
247 +
248 +
249 +class Resampler2(BaseResampler):
250 +
251 + def __init__(
252 + self,
253 + grid_size: int,
254 + embed_dim: int,
255 + num_heads: int,
256 + kv_dim: Optional[int] = None,
257 + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
258 + adaptive: bool = False,
259 + ) -> None:
260 + super().__init__(grid_size**2, embed_dim, num_heads, kv_dim,
261 + norm_layer)
262 +
263 + self.adaptive = adaptive
264 +
265 + pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
266 + grid_size,
267 + version=(2, 0))
268 + self.pos_embed = nn.Parameter(
269 + torch.from_numpy(pos_embed_arr).float()).requires_grad_(False)
270 +
271 + self.apply(self._init_weights)
272 +
273 + def forward(
274 + self,
275 + x: torch.Tensor,
276 + tgt_sizes: torch.Tensor,
277 + attn_mask: Optional[torch.Tensor] = None,
278 + ):
279 + if self.adaptive:
280 + pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
281 + tgt_sizes,
282 + version=(2, 0))
283 + pos_embed = torch.from_numpy(pos_embed_arr).to(device=x.device,
284 + dtype=x.dtype)
285 else:
211 - self._set_2d_pos_cache(self.max_size)
286 + pos_embed = get_abs_pos(self.pos_embed, tgt_sizes)
287 +
288 + x, _ = self.kv_proj(x)
289 + x = self.ln_kv(x).permute(1, 0, 2)
290 +
291 + N = x.shape[1]
292 + q = self.ln_q(self.query)
293 + out = self.attn(
294 + self._repeat(q, N) + self.pos_embed.unsqueeze(1),
295 + x + pos_embed.unsqueeze(1),
296 + x,
297 + attn_mask=attn_mask,
298 + )[0]
299 + x = out.permute(1, 0, 2)
300 +
301 + x = self.ln_post(x)
302 + x = x @ self.proj
303 + return x
304 +
305 +
306 +class Resampler2_5(BaseResampler):
307 +
308 + def __init__(
309 + self,
310 + num_queries: int,
311 + embed_dim: int,
312 + num_heads: int,
313 + kv_dim: Optional[int] = None,
314 + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
315 + max_size: Tuple[int, int] = (70, 70),
316 + ) -> None:
317 + super().__init__(num_queries, embed_dim, num_heads, kv_dim, norm_layer)
318 +
319 + self.max_size = max_size
320 + self._set_2d_pos_cache(self.max_size)
321
322 self.apply(self._init_weights)
323
324 def _set_2d_pos_cache(self,
325 max_size: Tuple[int, int],
217 - device: torch.types.Device = 'cpu'):
218 - pos_embed = torch.from_numpy(
219 - get_2d_sincos_pos_embed(self.embed_dim,
220 - max_size,
221 - version=self.version)).float().to(device)
326 + device: torch.types.Device = "cpu") -> None:
327 + pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
328 + max_size,
329 + version=(2, 5))
330 + pos_embed = torch.from_numpy(pos_embed_arr).float().to(device)
331 self.register_buffer("pos_embed", pos_embed, persistent=False)
332
333 def _adjust_pos_cache(self, tgt_sizes: torch.Tensor,
225 - device: torch.types.Device):
226 - max_h = torch.max(tgt_sizes[:, 0])
227 - max_w = torch.max(tgt_sizes[:, 1])
334 + device: torch.types.Device) -> None:
335 + max_h = tgt_sizes[:, 0].max().item()
336 + max_w = tgt_sizes[:, 1].max().item()
337 + assert isinstance(max_h, int) and isinstance(max_w, int)
338 +
339 if max_h > self.max_size[0] or max_w > self.max_size[1]:
229 - self.max_size = [
340 + self.max_size = (
341 max(max_h, self.max_size[0]),
231 - max(max_w, self.max_size[1])
232 - ]
342 + max(max_w, self.max_size[1]),
343 + )
344 self._set_2d_pos_cache(self.max_size, device)
345
235 - def _init_weights(self, m: nn.Module):
236 - if isinstance(m, nn.Linear):
237 - trunc_normal_(m.weight, std=.02)
238 - if isinstance(m, nn.Linear) and m.bias is not None:
239 - nn.init.constant_(m.bias, 0)
240 - elif isinstance(m, nn.LayerNorm):
241 - nn.init.constant_(m.bias, 0)
242 - nn.init.constant_(m.weight, 1.0)
243 -
244 - def forward_2_5(self,
245 - x: torch.Tensor,
246 - tgt_sizes: Optional[torch.Tensor] = None):
346 + def forward(self, x: torch.Tensor,
347 + tgt_sizes: torch.Tensor) -> torch.Tensor:
348 assert x.shape[0] == tgt_sizes.shape[0]
349 bs = x.shape[0]
350
@@ -254,25 +355,25 @@ class Resampler(nn.Module):
355
356 self._adjust_pos_cache(tgt_sizes, device=device)
357
257 - max_patch_len = torch.max(patch_len)
358 + max_patch_len = patch_len.max().item()
359 + assert isinstance(max_patch_len, int)
360 +
361 key_padding_mask = torch.zeros((bs, max_patch_len),
362 dtype=torch.bool,
363 device=device)
364
365 pos_embed = []
366 for i in range(bs):
264 - tgt_h, tgt_w = tgt_sizes[i]
367 + tgt_h, tgt_w = tgt_sizes[i].tolist()
368 pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape(
369 (tgt_h * tgt_w, -1)).to(dtype)) # patches * D
370 key_padding_mask[i, patch_len[i]:] = True
268 -
371 pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed,
372 batch_first=True,
373 padding_value=0.0).permute(
374 1, 0,
375 2) # BLD => L * B * D
274 -
275 - x = self.kv_proj(x) # B * L * D
376 + x, _ = self.kv_proj(x) # B * L * D
377 x = self.ln_kv(x).permute(1, 0, 2) # L * B * D
378
379 q = self.ln_q(self.query) # Q * D
@@ -281,7 +382,8 @@ class Resampler(nn.Module):
382 self._repeat(q, bs), # Q * B * D
383 x + pos_embed, # L * B * D + L * B * D
384 x,
284 - key_padding_mask=key_padding_mask)[0]
385 + key_padding_mask=key_padding_mask,
386 + )[0]
387 # out: Q * B * D
388 x = out.permute(1, 0, 2) # B * Q * D
389
@@ -289,45 +391,6 @@ class Resampler(nn.Module):
391 x = x @ self.proj
392 return x
393
292 - def forward_2(self,
293 - x: torch.Tensor,
294 - tgt_sizes: Optional[torch.Tensor] = None,
295 - attn_mask: Optional[torch.Tensor] = None):
296 - if self.adaptive:
297 - pos_embed = torch.Tensor(
298 - get_2d_sincos_pos_embed(self.embed_dim,
299 - tgt_sizes)).float().to(device=x.device,
300 - dtype=x.dtype)
301 - else:
302 - pos_embed = get_abs_pos(self.pos_embed, tgt_sizes)
303 -
304 - x = self.kv_proj(x)
305 - x = self.ln_kv(x).permute(1, 0, 2)
306 -
307 - N = x.shape[1]
308 - q = self.ln_q(self.query)
309 - out = self.attn(self._repeat(q, N) + self.pos_embed.unsqueeze(1),
310 - x + pos_embed.unsqueeze(1),
311 - x,
312 - attn_mask=attn_mask)[0]
313 - x = out.permute(1, 0, 2)
314 -
315 - x = self.ln_post(x)
316 - x = x @ self.proj
317 - return x
318 -
319 - def forward(self,
320 - x: torch.Tensor,
321 - tgt_sizes: Optional[torch.Tensor] = None,
322 - attn_mask: Optional[torch.Tensor] = None):
323 - if self.version == (2, 0):
324 - return self.forward_2(x, tgt_sizes=tgt_sizes, attn_mask=attn_mask)
325 - else:
326 - return self.forward_2_5(x, tgt_sizes=tgt_sizes)
327 -
328 - def _repeat(self, query, N: int):
329 - return query.unsqueeze(1).repeat(1, N, 1)
330 -
394
395 def get_max_minicpmv_image_tokens(ctx: InputContext):
396 hf_config = ctx.get_hf_config(PretrainedConfig)
@@ -348,10 +411,7 @@ def dummy_image_for_minicpmv(hf_config: PretrainedConfig):
411 def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int):
412 hf_config = ctx.get_hf_config(PretrainedConfig)
413
351 - # image_feature_size = get_max_minicpmv_image_tokens(ctx)
352 -
414 seq_data = dummy_seq_data_for_minicpmv(seq_len)
354 -
415 mm_data = dummy_image_for_minicpmv(hf_config)
416
417 return seq_data, mm_data
@@ -376,25 +436,36 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
436 pattern = "(<image>./</image>)"
437 image = multi_modal_data["image"]
438 image_tags = re.findall(pattern, prompt)
379 - assert len(image_tags) <= 1
380 - text_chunks = prompt.split(pattern)
381 - new_prompt = text_chunks[0] \
382 - + image_processor.get_slice_image_placeholder(image.size) \
383 - + text_chunks[1]
439
385 - new_token_ids = tokenizer.encode(new_prompt)
386 -
387 - llm_inputs = LLMInputs(prompt_token_ids=new_token_ids,
388 - prompt=new_prompt,
389 - multi_modal_data=multi_modal_data)
440 + if len(image_tags) == 0:
441 + new_token_ids = token_ids
442 + new_prompt = prompt
443 + else:
444 + if len(image_tags) > 1:
445 + logger.warning("Multiple image input is not supported yet, "
446 + "so any extra image tokens will be treated "
447 + "as plain text.")
448 +
449 + text_chunks = prompt.split(pattern)
450 + new_prompt = (text_chunks[0] +
451 + image_processor.get_slice_image_placeholder(image.size) +
452 + "".join(text_chunks[1:]))
453 +
454 + new_token_ids = tokenizer.encode(new_prompt)
455 +
456 + llm_inputs = LLMInputs(
457 + prompt_token_ids=new_token_ids,
458 + prompt=new_prompt,
459 + multi_modal_data=multi_modal_data,
460 + )
461 return llm_inputs
462
463
393 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
394 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
395 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
396 -@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
397 -class MiniCPMV(nn.Module, SupportsVision):
464 +class MiniCPMVBaseModel(nn.Module, SupportsVision):
465 + """
466 + The abstract class of MiniCPMV can only be inherited, but cannot be
467 + instantiated.
468 + """
469
470 def __init__(
471 self,
@@ -419,8 +490,8 @@ class MiniCPMV(nn.Module, SupportsVision):
490 self.vpm = self.init_vision_module()
491 param_dtype = torch.get_default_dtype()
492 self.vpm.to(dtype=param_dtype)
422 - self.vision_dim = self.vpm.embed_dim if self.version == (2, 0) \
423 - else self.vpm.embeddings.embed_dim
493 + self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
494 + self.vpm.embeddings.embed_dim)
495 self.embed_dim = self.config.hidden_size
496 self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
497 self.resampler.to(device="cuda", dtype=param_dtype)
@@ -430,248 +501,100 @@ class MiniCPMV(nn.Module, SupportsVision):
501 self.logits_processor = LogitsProcessor(config.vocab_size)
502 self.sampler = Sampler()
503
433 - def init_llm(self,
434 - config: PretrainedConfig,
435 - cache_config: Optional[CacheConfig] = None,
436 - quant_config: Optional[QuantizationConfig] = None):
437 - if self.version == (2, 0):
438 - return MiniCPMModel(config,
439 - cache_config=cache_config,
440 - quant_config=quant_config)
441 - elif self.version == (2, 5):
442 - return LlamaModel(config,
443 - cache_config=cache_config,
444 - quant_config=quant_config)
445 - else:
446 - return Qwen2Model(config,
447 - cache_config=cache_config,
448 - quant_config=quant_config)
449 -
450 - def init_vision_module(self):
451 - if self.version == (2, 0):
452 - try:
453 - import timm
454 - except ImportError:
455 - raise ImportError(
456 - 'Please install timm==0.9.10') from ImportError
457 - default_dtype = torch.get_default_dtype()
458 - torch.set_default_dtype(torch.float16)
459 - model = timm.create_model('vit_so400m_patch14_siglip_384.webli',
460 - pretrained=False,
461 - num_classes=0,
462 - dynamic_img_size=True,
463 - dynamic_img_pad=True)
464 - torch.set_default_dtype(default_dtype)
465 - if isinstance(model, timm.models.VisionTransformer
466 - ) and model.attn_pool is not None:
467 - model.attn_pool = torch.nn.Identity()
468 -
469 - if self.config.drop_vision_last_layer:
470 - model.blocks = model.blocks[:-1]
471 - elif self.version == (2, 5):
472 - from transformers.models.idefics2.modeling_idefics2 import (
473 - Idefics2VisionTransformer)
474 - model = Idefics2VisionTransformer(self.config.vision_config)
475 - if self.config.drop_vision_last_layer:
476 - model.encoder.layers = model.encoder.layers[:-1]
477 - else:
478 - from vllm.model_executor.models.na_vit import (
479 - SiglipVisionTransformer)
480 - if self.config._attn_implementation == 'flash_attention_2':
481 - self.config.vision_config._attn_implementation \
482 - = 'flash_attention_2'
483 - else:
484 - # not support sdpa
485 - self.config.vision_config._attn_implementation = 'eager'
486 - model = SiglipVisionTransformer(self.config.vision_config)
487 - if self.config.drop_vision_last_layer:
488 - model.encoder.layers = model.encoder.layers[:-1]
489 - return model
490 -
491 - def init_resampler(self, embed_dim: int, vision_dim: int):
492 - default_dtype = torch.get_default_dtype()
493 - torch.set_default_dtype(torch.float16)
494 - if self.version == (2, 0):
495 - resampler = Resampler(grid_size=int(
496 - math.sqrt(self.config.query_num)),
497 - num_queries=None,
498 - embed_dim=embed_dim,
499 - num_heads=embed_dim // 128,
500 - kv_dim=vision_dim,
501 - adaptive=True,
502 - version=self.version)
504 + def get_embedding(
505 + self,
506 + input_ids: torch.Tensor,
507 + image_inputs: Optional[MiniCPMVImageInputs],
508 + ) -> Tuple[torch.Tensor, torch.Tensor]:
509 + vlm_embedding: torch.Tensor = self.llm.embed_tokens(input_ids)
510 + if hasattr(self.config, "scale_emb"):
511 + vlm_embedding *= self.config.scale_emb
512 +
513 + if image_inputs is None: # No image
514 + vision_hidden_states = torch.tensor([], device=input_ids.device)
515 else:
504 - resampler = Resampler(num_queries=self.config.query_num,
505 - grid_size=None,
506 - embed_dim=embed_dim,
507 - num_heads=embed_dim // 128,
508 - kv_dim=vision_dim,
509 - adaptive=True,
510 - version=self.version)
511 - torch.set_default_dtype(default_dtype)
512 - return resampler
516 + vision_hidden_states = self.get_vision_hidden_states(image_inputs)
517 +
518 + # See NOTE in _parse_and_validate_inputs
519 + image_bounds = image_inputs["image_bounds"]
520 + if len(image_bounds) > 0:
521 + image_indices = torch.stack([
522 + torch.arange(start, end, dtype=torch.long)
523 + for start, end in image_bounds.tolist()
524 + ]).to(vlm_embedding.device)
525 + vlm_embedding.scatter_(
526 + 0,
527 + image_indices.view(-1, 1).repeat(1,
528 + vlm_embedding.shape[-1]),
529 + vision_hidden_states.view(-1,
530 + vision_hidden_states.shape[-1]),
531 + )
532
514 - def get_vision_embedding(self,
515 - pixel_values: List[List[torch.Tensor]],
516 - patch_attn_mask: Optional[torch.Tensor] = None,
517 - tgt_sizes: Optional[torch.Tensor] = None,
518 - version: Tuple[int, int] = (2, 0)):
519 - if version == (2, 0):
520 - res = []
521 - dtype = self.vpm.pos_embed.data.dtype
522 - for pixel_value in pixel_values:
523 - # V2.0 start
524 - H, W = pixel_value[0].shape[-2:]
525 - tgt_size = (math.ceil(H / self.vpm.patch_embed.patch_size[0]),
526 - math.ceil(W / self.vpm.patch_embed.patch_size[0]))
527 - # V2.0 end
528 - vision_embedding = self.vpm.forward_features(
529 - pixel_value.unsqueeze(0).type(dtype))
530 - if hasattr(self.vpm, 'num_prefix_tokens'
531 - ) and self.vpm.num_prefix_tokens > 0:
532 - vision_embedding = vision_embedding[:, self.vpm.
533 - num_prefix_tokens:]
534 - res.append(self.resampler(vision_embedding, tgt_size))
535 - return torch.vstack(res)
536 - elif version == (2, 5):
537 - vision_embedding = self.vpm(
538 - pixel_values.type(dtype),
539 - patch_attention_mask=patch_attn_mask).last_hidden_state
540 - vision_embedding = self.resampler(vision_embedding, tgt_sizes)
541 - else:
542 - vision_embedding = self.vpm(pixel_values.type(dtype),
543 - patch_attention_mask=patch_attn_mask,
544 - tgt_sizes=tgt_sizes).last_hidden_state
533 + return vlm_embedding, vision_hidden_states
534
546 - def get_image_bounds(self, input_ids: torch.Tensor):
535 + def _get_image_bounds(self, input_ids: torch.Tensor) -> torch.Tensor:
536 tokenizer = cached_get_tokenizer(self.config._name_or_path,
537 trust_remote_code=True)
549 - if not hasattr(tokenizer, "slice_start_id"):
550 - start_cond = input_ids == tokenizer.im_start_id
551 - end_cond = input_ids == tokenizer.im_end_id
552 - else:
553 - start_cond = (input_ids == tokenizer.im_start_id) | (
554 - input_ids == tokenizer.slice_start_id)
555 - end_cond = (input_ids == tokenizer.im_end_id) | (
556 - input_ids == tokenizer.slice_end_id)
538 + start_cond = input_ids == tokenizer.im_start_id
539 + end_cond = input_ids == tokenizer.im_end_id
540 + if hasattr(tokenizer, "slice_start_id"):
541 + start_cond |= (input_ids == tokenizer.slice_start_id)
542 + end_cond |= (input_ids == tokenizer.slice_end_id)
543
558 - image_start_tokens = torch.where(start_cond)[0]
544 + image_start_tokens, = torch.where(start_cond)
545 image_start_tokens += 1
560 - image_end_tokens = torch.where(end_cond)[0]
546 + image_end_tokens, = torch.where(end_cond)
547 valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))
548 +
549 if valid_image_nums == 0:
563 - return []
564 - image_bound = torch.hstack([
550 + return torch.zeros((0, 2), device=input_ids.device)
551 +
552 + return torch.hstack([
553 image_start_tokens[:valid_image_nums].unsqueeze(-1),
554 image_end_tokens[:valid_image_nums].unsqueeze(-1),
555 ])
556
569 - return image_bound
570 -
571 - def get_vision_hidden_states(self, data: Dict[str,
572 - Union[List[torch.Tensor],
573 - torch.Tensor]]):
574 - if "vision_hidden_states" not in data:
575 - pixel_values = data["pixel_values"]
576 - tgt_sizes = data["tgt_sizes"]
577 - vision_hidden_states = []
578 - if self.version == (2, 0):
579 - if pixel_values is not None and len(pixel_values) > 0:
580 - vision_hidden_states = self.get_vision_embedding(
581 - pixel_values)
582 - else:
583 - vision_hidden_states = torch.tensor([]).to(
584 - data["input_ids"].device)
585 - else:
586 - device = self.vpm.embeddings.position_embedding.weight.device
587 - dtype = self.vpm.embeddings.position_embedding.weight.dtype
588 - all_pixel_values = [
589 - i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
590 - ]
591 - if all_pixel_values:
592 - tgt_sizes = torch.vstack(tgt_sizes).type(torch.int32)
593 - max_patches = torch.max(tgt_sizes[:, 0] * tgt_sizes[:, 1])
594 - all_pixel_values = torch.nn.utils.rnn.pad_sequence(
595 - all_pixel_values, batch_first=True, padding_value=0.0)
596 - B, L, _ = all_pixel_values.shape
597 - all_pixel_values = all_pixel_values.permute(
598 - 0, 2, 1).reshape(B, 3, -1, L)
599 - patch_attn_mask = torch.zeros((B, 1, max_patches),
600 - dtype=torch.bool,
601 - device=device)
602 - if self.version == (2, 5):
603 - for i in range(B):
604 - patch_attn_mask[i, :tgt_sizes[i][0] *
605 - tgt_sizes[i][1]] = True
606 - vision_embedding = self.vpm(
607 - all_pixel_values.type(dtype),
608 - patch_attention_mask=patch_attn_mask
609 - ).last_hidden_state
610 - else:
611 - for i in range(B):
612 - patch_attn_mask[i, 0, :tgt_sizes[i][0] *
613 - tgt_sizes[i][1]] = True
614 - vision_embedding = self.vpm(
615 - all_pixel_values.type(dtype),
616 - patch_attention_mask=patch_attn_mask,
617 - tgt_sizes=tgt_sizes).last_hidden_state
618 -
619 - vision_hidden_states = self.resampler(
620 - vision_embedding, tgt_sizes)
621 -
622 - else: # no image
623 - dummy_feature = []
624 - vision_hidden_states = dummy_feature
625 - else:
626 - vision_hidden_states = data["vision_hidden_states"]
627 -
628 - return vision_hidden_states
629 -
630 - def get_embedding(self, data: Dict[str, Union[List[torch.Tensor],
631 - torch.Tensor]]):
632 - input_ids = data["input_ids"]
633 -
634 - vision_hidden_states = self.get_vision_hidden_states(data)
635 - if vision_hidden_states is not None and len(vision_hidden_states) > 0:
636 - image_bounds = self.get_image_bounds(input_ids)
637 - else:
638 - image_bounds = []
639 -
640 - if hasattr(self.config, 'scale_emb'):
641 - vlm_embedding = self.llm.embed_tokens(
642 - input_ids) * self.config.scale_emb
643 - else:
644 - vlm_embedding = self.llm.embed_tokens(input_ids)
645 - vision_hidden_states = [
646 - i.type(vlm_embedding.dtype) if isinstance(i, torch.Tensor) else i
647 - for i in vision_hidden_states
648 - ]
649 -
650 - if len(vision_hidden_states) > 0 and len(image_bounds) > 0:
651 - vision_hidden_states = torch.cat(vision_hidden_states, dim=0)
652 - image_indices = torch.stack([
653 - torch.arange(r[0], r[1], dtype=torch.long)
654 - for r in image_bounds
655 - ]).to(vlm_embedding.device)
656 - vlm_embedding.scatter_(
657 - 0,
658 - image_indices.view(-1, 1).repeat(1, vlm_embedding.shape[-1]),
659 - vision_hidden_states.view(-1, vision_hidden_states.shape[-1]))
660 - return vlm_embedding, vision_hidden_states
661 -
662 - def process_multimodal_inputs(self, inputs: Dict[str,
663 - Union[List[torch.Tensor],
664 - torch.Tensor]]):
665 - pixel_values = []
666 - tgt_sizes = []
667 - for b in range(len(inputs["pixel_values"])):
668 - pixel_values += inputs["pixel_values"][b]
669 - tgt_sizes += inputs["tgt_sizes"][b]
670 - return {
671 - "pixel_values": pixel_values,
672 - "input_ids": inputs["input_ids"],
673 - "tgt_sizes": tgt_sizes
674 - }
557 + def _parse_and_validate_inputs(
558 + self,
559 + input_ids: torch.Tensor,
560 + **kwargs: object,
561 + ) -> Optional[MiniCPMVImageInputs]:
562 + pixel_values = kwargs.pop("pixel_values", [])
563 + tgt_sizes = kwargs.pop("tgt_sizes", [])
564 +
565 + if not isinstance(pixel_values, (torch.Tensor, list)):
566 + raise ValueError("Incorrect type of pixel values. "
567 + f"Got type: {type(pixel_values)}")
568 +
569 + if not isinstance(tgt_sizes, (torch.Tensor, list)):
570 + raise ValueError("Incorrect type of target sizes. "
571 + f"Got type: {type(tgt_sizes)}")
572 +
573 + if len(pixel_values) != len(tgt_sizes):
574 + raise ValueError("Inconsistent batch lengths, found: "
575 + f"{len(pixel_values)} vs. {len(tgt_sizes)}")
576 +
577 + pixel_values_flat: List[torch.Tensor] = []
578 + tgt_sizes_flat: List[torch.Tensor] = []
579 + for b in range(len(pixel_values)):
580 + pixel_values_flat += pixel_values[b]
581 + tgt_sizes_flat += tgt_sizes[b]
582 +
583 + # NOTE: Input IDs does not contain image tokens during memory profiling,
584 + # so we allow it to be empty
585 + if len(pixel_values_flat) != len(tgt_sizes_flat):
586 + raise ValueError("Inconsistent flattened lengths, found: "
587 + f"{len(pixel_values_flat)} vs. "
588 + f"{len(tgt_sizes_flat)}")
589 +
590 + if len(pixel_values_flat) == 0:
591 + return None
592 +
593 + return MiniCPMVImageInputs(
594 + image_bounds=self._get_image_bounds(input_ids),
595 + pixel_values=pixel_values_flat,
596 + tgt_sizes=torch.stack(tgt_sizes_flat),
597 + )
598
599 def forward(
600 self,
@@ -680,23 +603,20 @@ class MiniCPMV(nn.Module, SupportsVision):
603 kv_caches: List[torch.Tensor],
604 attn_metadata: AttentionMetadata,
605 intermediate_tensors: Optional[IntermediateTensors] = None,
683 - **kwargs: object,
684 - ):
685 - inputs = {
686 - "pixel_values": kwargs.pop("pixel_values", []),
687 - "input_ids": input_ids,
688 - "tgt_sizes": kwargs.pop("tgt_sizes", None),
689 - }
690 - inputs = self.process_multimodal_inputs(inputs)
691 -
692 - vlm_embeddings, vision_hidden_states = self.get_embedding(inputs)
693 -
694 - output = self.llm(input_ids=None,
695 - positions=positions,
696 - kv_caches=kv_caches,
697 - attn_metadata=attn_metadata,
698 - intermediate_tensors=intermediate_tensors,
699 - inputs_embeds=vlm_embeddings)
606 + **kwargs: Any,
607 + ) -> torch.Tensor:
608 + image_inputs = self._parse_and_validate_inputs(input_ids, **kwargs)
609 +
610 + vlm_embeddings, _ = self.get_embedding(input_ids, image_inputs)
611 +
612 + output = self.llm(
613 + input_ids=None,
614 + positions=positions,
615 + kv_caches=kv_caches,
616 + attn_metadata=attn_metadata,
617 + intermediate_tensors=intermediate_tensors,
618 + inputs_embeds=vlm_embeddings,
619 + )
620 return output
621
622 def compute_logits(self, hidden_states: torch.Tensor,
@@ -735,13 +655,10 @@ class MiniCPMV(nn.Module, SupportsVision):
655 # the checkpoint. Skip them.
656 continue
657 use_default_weight_loading = False
738 - if "vpm" in name or 'resampler' in name:
739 - # We only do sharding for language model and
740 - # not vision model for now.
658 + if self.is_default_weight_loading(name):
659 use_default_weight_loading = True
660 else:
743 - for (param_name, weight_name,
744 - shard_id) in stacked_params_mapping:
661 + for param_name, weight_name, shard_id in stacked_params_mapping:
662 if weight_name not in name:
663 continue
664 param = params_dict[name.replace(weight_name, param_name)]
@@ -755,3 +672,341 @@ class MiniCPMV(nn.Module, SupportsVision):
672 weight_loader = getattr(param, "weight_loader",
673 default_weight_loader)
674 weight_loader(param, loaded_weight)
675 +
676 + def init_llm(
677 + self,
678 + config: PretrainedConfig,
679 + cache_config: Optional[CacheConfig] = None,
680 + quant_config: Optional[QuantizationConfig] = None,
681 + ) -> nn.Module:
682 + raise NotImplementedError
683 +
684 + def init_vision_module(self) -> nn.Module:
685 + raise NotImplementedError
686 +
687 + def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
688 + raise NotImplementedError
689 +
690 + def get_vision_embedding(
691 + self,
692 + pixel_values: List[torch.Tensor],
693 + patch_attn_mask: Optional[torch.Tensor] = None,
694 + tgt_sizes: Optional[torch.Tensor] = None,
695 + ) -> torch.Tensor:
696 + raise NotImplementedError
697 +
698 + def get_vision_hidden_states(self,
699 + data: MiniCPMVImageInputs) -> torch.Tensor:
700 + raise NotImplementedError
701 +
702 + def is_default_weight_loading(self, name: str) -> bool:
703 + raise NotImplementedError
704 +
705 +
706 +class MiniCPMV2(MiniCPMVBaseModel):
707 +
708 + def __init__(
709 + self,
710 + config: PretrainedConfig,
711 + multimodal_config: MultiModalConfig,
712 + cache_config: Optional[CacheConfig] = None,
713 + quant_config: Optional[QuantizationConfig] = None,
714 + ):
715 + super().__init__(config, multimodal_config, cache_config, quant_config)
716 + assert self.version == (2, 0)
717 +
718 + def init_llm(
719 + self,
720 + config: PretrainedConfig,
721 + cache_config: Optional[CacheConfig] = None,
722 + quant_config: Optional[QuantizationConfig] = None,
723 + ) -> nn.Module:
724 + return MiniCPMModel(config,
725 + cache_config=cache_config,
726 + quant_config=quant_config)
727 +
728 + def init_vision_module(self) -> nn.Module:
729 + # TODO :refactor this vision model
730 + try:
731 + import timm
732 + except ImportError:
733 + raise ImportError("Please install timm==0.9.10") from ImportError
734 + with set_default_torch_dtype(torch.float16):
735 + model = timm.create_model(
736 + "vit_so400m_patch14_siglip_384.webli",
737 + pretrained=False,
738 + num_classes=0,
739 + dynamic_img_size=True,
740 + dynamic_img_pad=True,
741 + )
742 +
743 + if (isinstance(model, timm.models.VisionTransformer)
744 + and model.attn_pool is not None):
745 + model.attn_pool = torch.nn.Identity()
746 +
747 + if self.config.drop_vision_last_layer:
748 + model.blocks = model.blocks[:-1]
749 +
750 + return model
751 +
752 + def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
753 + with set_default_torch_dtype(torch.float16):
754 + resampler = Resampler2(
755 + embed_dim=embed_dim,
756 + num_heads=embed_dim // 128,
757 + grid_size=int(math.sqrt(self.config.query_num)),
758 + kv_dim=vision_dim,
759 + adaptive=True,
760 + )
761 +
762 + return resampler
763 +
764 + def get_vision_embedding(
765 + self,
766 + pixel_values: List[torch.Tensor],
767 + patch_attn_mask: Optional[torch.Tensor] = None,
768 + tgt_sizes: Optional[torch.Tensor] = None,
769 + ) -> torch.Tensor:
770 + res = []
771 + dtype = self.vpm.pos_embed.data.dtype
772 + for pixel_value in pixel_values:
773 + H, W = pixel_value[0].shape[-2:]
774 + tgt_size = (
775 + math.ceil(H / self.vpm.patch_embed.patch_size[0]),
776 + math.ceil(W / self.vpm.patch_embed.patch_size[0]),
777 + )
778 + vision_embedding = self.vpm.forward_features(
779 + pixel_value.unsqueeze(0).type(dtype))
780 + if (hasattr(self.vpm, "num_prefix_tokens")
781 + and self.vpm.num_prefix_tokens > 0):
782 + vision_embedding = vision_embedding[:, self.vpm.
783 + num_prefix_tokens:]
784 + res.append(self.resampler(vision_embedding, tgt_size))
785 + return torch.vstack(res)
786 +
787 + def get_vision_hidden_states(self,
788 + data: MiniCPMVImageInputs) -> torch.Tensor:
789 + pixel_values = data["pixel_values"]
790 +
791 + return self.get_vision_embedding(pixel_values)
792 +
793 + def is_default_weight_loading(self, name: str) -> bool:
794 + return "resampler" in name or "vpm" in name
795 +
796 +
797 +class MiniCPMV2_5(MiniCPMVBaseModel):
798 +
799 + def __init__(
800 + self,
801 + config: PretrainedConfig,
802 + multimodal_config: MultiModalConfig,
803 + cache_config: Optional[CacheConfig] = None,
804 + quant_config: Optional[QuantizationConfig] = None,
805 + ):
806 + super().__init__(config, multimodal_config, cache_config, quant_config)
807 + assert self.version == (2, 5)
808 +
809 + def init_llm(
810 + self,
811 + config: PretrainedConfig,
812 + cache_config: Optional[CacheConfig] = None,
813 + quant_config: Optional[QuantizationConfig] = None,
814 + ) -> nn.Module:
815 + return LlamaModel(config,
816 + cache_config=cache_config,
817 + quant_config=quant_config)
818 +
819 + def init_vision_module(self) -> nn.Module:
820 + model = Idefics2VisionTransformer(self.config.vision_config)
821 + if self.config.drop_vision_last_layer:
822 + model.encoder.layers = model.encoder.layers[:-1]
823 + return model
824 +
825 + def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
826 + with set_default_torch_dtype(torch.float16):
827 + resampler = Resampler2_5(
828 + num_queries=self.config.query_num,
829 + embed_dim=embed_dim,
830 + num_heads=embed_dim // 128,
831 + kv_dim=vision_dim,
832 + )
833 + return resampler
834 +
835 + def get_vision_embedding(
836 + self,
837 + pixel_values: List[torch.Tensor],
838 + patch_attn_mask: Optional[torch.Tensor] = None,
839 + tgt_sizes: Optional[torch.Tensor] = None,
840 + ) -> torch.Tensor:
841 + vision_embedding = self.vpm(pixel_values,
842 + patch_attention_mask=patch_attn_mask)
843 + vision_embedding = self.resampler(vision_embedding, tgt_sizes)
844 + return vision_embedding
845 +
846 + def get_vision_hidden_states(self,
847 + data: MiniCPMVImageInputs) -> torch.Tensor:
848 + pixel_values = data["pixel_values"]
849 + tgt_sizes = data["tgt_sizes"]
850 +
851 + device = self.vpm.embeddings.position_embedding.weight.device
852 + dtype = self.vpm.embeddings.position_embedding.weight.dtype
853 + all_pixel_values_lst = [
854 + i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
855 + ]
856 +
857 + max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item()
858 + assert isinstance(max_patches, int)
859 +
860 + all_pixel_values = torch.nn.utils.rnn.pad_sequence(
861 + all_pixel_values_lst, batch_first=True, padding_value=0.0)
862 + B, L, _ = all_pixel_values.shape
863 + all_pixel_values = all_pixel_values.permute(0, 2,
864 + 1).reshape(B, 3, -1, L)
865 +
866 + patch_attn_mask = torch.zeros((B, 1, max_patches),
867 + dtype=torch.bool,
868 + device=device)
869 + for i in range(B):
870 + patch_attn_mask[i, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True
871 +
872 + return self.get_vision_embedding(all_pixel_values.type(dtype),
873 + patch_attn_mask, tgt_sizes)
874 +
875 + def is_default_weight_loading(self, name: str) -> bool:
876 + return "resampler" in name
877 +
878 +
879 +# NOTE: Currently, information about this model is unavailable. We are
880 +# temporarily using `MiniCPMVQwen2` as it's name. The name may need
881 +# to be modified in the future.
882 +class MiniCPMVQwen2(MiniCPMVBaseModel):
883 +
884 + def __init__(
885 + self,
886 + config: PretrainedConfig,
887 + multimodal_config: MultiModalConfig,
888 + cache_config: Optional[CacheConfig] = None,
889 + quant_config: Optional[QuantizationConfig] = None,
890 + ):
891 + super().__init__(config, multimodal_config, cache_config, quant_config)
892 +
893 + def init_llm(
894 + self,
895 + config: PretrainedConfig,
896 + cache_config: Optional[CacheConfig] = None,
897 + quant_config: Optional[QuantizationConfig] = None,
898 + ) -> nn.Module:
899 + return Qwen2Model(config,
900 + cache_config=cache_config,
901 + quant_config=quant_config)
902 +
903 + def init_vision_module(self) -> nn.Module:
904 + # A custom version of SiglipVisionTransformer, won't work with TP
905 + from vllm.model_executor.models.na_vit import SiglipVisionTransformer
906 +
907 + if self.config._attn_implementation == "flash_attention_2":
908 + self.config.vision_config._attn_implementation = "flash_attention_2"
909 + else:
910 + # not support sdpa
911 + self.config.vision_config._attn_implementation = "eager"
912 + model = SiglipVisionTransformer(self.config.vision_config)
913 + if self.config.drop_vision_last_layer:
914 + model.encoder.layers = model.encoder.layers[:-1]
915 + return model
916 +
917 + def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
918 + with set_default_torch_dtype(torch.float16):
919 + resampler = Resampler2_5(
920 + num_queries=self.config.query_num,
921 + embed_dim=embed_dim,
922 + num_heads=embed_dim // 128,
923 + kv_dim=vision_dim,
924 + )
925 +
926 + return resampler
927 +
928 + def get_vision_embedding(
929 + self,
930 + pixel_values: List[torch.Tensor],
931 + patch_attn_mask: Optional[torch.Tensor] = None,
932 + tgt_sizes: Optional[torch.Tensor] = None,
933 + ) -> torch.Tensor:
934 + vision_embedding = self.vpm(
935 + pixel_values,
936 + patch_attention_mask=patch_attn_mask,
937 + tgt_sizes=tgt_sizes,
938 + ).last_hidden_state
939 + return vision_embedding
940 +
941 + def get_vision_hidden_states(self,
942 + data: MiniCPMVImageInputs) -> torch.Tensor:
943 + pixel_values = data["pixel_values"]
944 + tgt_sizes = data["tgt_sizes"]
945 +
946 + device = self.vpm.embeddings.position_embedding.weight.device
947 + dtype = self.vpm.embeddings.position_embedding.weight.dtype
948 + all_pixel_values_lst = [
949 + i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
950 + ]
951 +
952 + max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item()
953 + assert isinstance(max_patches, int)
954 +
955 + all_pixel_values = torch.nn.utils.rnn.pad_sequence(
956 + all_pixel_values_lst, batch_first=True, padding_value=0.0)
957 + B, L, _ = all_pixel_values.shape
958 + all_pixel_values = all_pixel_values.permute(0, 2,
959 + 1).reshape(B, 3, -1, L)
960 +
961 + patch_attn_mask = torch.zeros((B, 1, max_patches),
962 + dtype=torch.bool,
963 + device=device)
964 + for i in range(B):
965 + patch_attn_mask[i, 0, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True
966 + vision_embedding = self.vpm(
967 + all_pixel_values.type(dtype),
968 + patch_attention_mask=patch_attn_mask,
969 + tgt_sizes=tgt_sizes,
970 + ).last_hidden_state
971 +
972 + return self.resampler(vision_embedding, tgt_sizes)
973 +
974 + def is_default_weight_loading(self, name: str) -> bool:
975 + return "resampler" in name or "vpm" in name
976 +
977 +
978 +@MULTIMODAL_REGISTRY.register_image_input_mapper()
979 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
980 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
981 +@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
982 +class MiniCPMV(MiniCPMVBaseModel):
983 + """
984 + Different versions of MiniCPMV use different visual encoders and LLMs,
985 + which is not conducive to the current integration logic of LoRA and
986 + bitsandbytes in vLLM. Therefore, it is necessary to separate them.
987 + """
988 +
989 + def __new__(
990 + cls,
991 + config: PretrainedConfig,
992 + multimodal_config: MultiModalConfig,
993 + cache_config: Optional[CacheConfig] = None,
994 + quant_config: Optional[QuantizationConfig] = None,
995 + ):
996 + if not hasattr(config, "version"):
997 + if config.hidden_size == 2304 and config.query_num == 64:
998 + version = (2, 0)
999 + else:
1000 + version = (2, 5)
1001 + else:
1002 + version = str(config.version).split(".")
1003 + version = tuple([int(x) for x in version])
1004 + # Dispatch class based on version
1005 + if version == (2, 0):
1006 + instance_class = MiniCPMV2
1007 + elif version == (2, 5):
1008 + instance_class = MiniCPMV2_5
1009 + else:
1010 + instance_class = MiniCPMVQwen2
1011 + return instance_class(config, multimodal_config, cache_config,
1012 + quant_config)
vllm/model_executor/models/na_vit.py
+1 -1
@@ -100,7 +100,7 @@ def _get_unpad_data(attention_mask):
100 indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
101 max_seqlen_in_batch = seqlens_in_batch.max().item()
102 cu_seqlens = F.pad(
103 - torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
103 + torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
104 return (
105 indices,
106 cu_seqlens,