A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model]Refactor MiniCPMV (#7020)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Jee Jee Li committed
Aug 4, 2024 at 16:12 UTC
179a6a36f2a585df49ce9c26701b1b9d894bd00e
4 files changed
+937
-386
docs/source/models/supported_models.rst
+1
-1
@@ -220,7 +220,7 @@ Vision Language Models
220
- Phi-3-Vision
221
- :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
222
-
223
- * - :code:`MiniCPM-V`
223
+ * - :code:`MiniCPMV`
224
- MiniCPM-V
225
- :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
226
-
vllm/model_executor/models/idefics2_vision_model.py
new
+296
@@ -0,0 +1,296 @@
1
+# coding=utf-8
2
+
3
+# adapted from https://github.com/huggingface/transformers/blob/v4.43.2/src/transformers/models/idefics2/modeling_idefics2.py
4
+# Copyright 2024 The vLLM team.
5
+# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# Licensed under the Apache License, Version 2.0 (the "License");
8
+# you may not use this file except in compliance with the License.
9
+# You may obtain a copy of the License at
10
+#
11
+# http://www.apache.org/licenses/LICENSE-2.0
12
+#
13
+# Unless required by applicable law or agreed to in writing, software
14
+# distributed under the License is distributed on an "AS IS" BASIS,
15
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+# See the License for the specific language governing permissions and
17
+# limitations under the License.
18
+"""PyTorch Idefics2 model."""
19
+
20
+from typing import Optional
21
+
22
+import torch
23
+from torch import nn
24
+from transformers.models.idefics2.configuration_idefics2 import (
25
+ Idefics2Config, Idefics2VisionConfig)
26
+from xformers import ops as xops
27
+
28
+from vllm.distributed import divide, get_tensor_model_parallel_world_size
29
+from vllm.model_executor.layers.activation import get_act_fn
30
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
+ QKVParallelLinear,
32
+ RowParallelLinear)
33
+from vllm.model_executor.layers.quantization import QuantizationConfig
34
+
35
+
36
+class Idefics2VisionEmbeddings(nn.Module):
37
+ """
38
+ This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings
39
+ ` to enable images of variable
40
+ resolution.
41
+
42
+ The modifications are adapted from [Patch n' Pack: NaViT, a Vision
43
+ Transformer for any Aspect Ratio and Resolution](https://arxiv.org/abs/2307.06304)
44
+ which allows treating images in their native aspect ratio and without the
45
+ need to resize them to the same fixed size. In particular, we start from the
46
+ original pre-trained SigLIP model(which uses images of fixed-size square
47
+ images) and adapt it by training on images of variable resolutions.
48
+ """
49
+
50
+ def __init__(self, config: Idefics2VisionConfig):
51
+ super().__init__()
52
+ self.embed_dim = config.hidden_size
53
+ self.image_size = config.image_size
54
+ self.patch_size = config.patch_size
55
+ self.patch_embedding = nn.Conv2d(
56
+ in_channels=config.num_channels,
57
+ out_channels=self.embed_dim,
58
+ kernel_size=self.patch_size,
59
+ stride=self.patch_size,
60
+ padding="valid",
61
+ )
62
+ self.num_patches_per_side = self.image_size // self.patch_size
63
+ self.num_patches = self.num_patches_per_side**2
64
+ self.num_positions = self.num_patches
65
+ self.position_embedding = nn.Embedding(self.num_positions,
66
+ self.embed_dim)
67
+
68
+ def forward(
69
+ self,
70
+ pixel_values: torch.FloatTensor,
71
+ patch_attention_mask: torch.BoolTensor,
72
+ ) -> torch.Tensor:
73
+ batch_size, _, max_im_h, max_im_w = pixel_values.shape
74
+ patch_embeds = self.patch_embedding(pixel_values)
75
+ embeddings = patch_embeds.flatten(2).transpose(1, 2)
76
+ max_nb_patches_h, max_nb_patches_w = (
77
+ max_im_h // self.patch_size,
78
+ max_im_w // self.patch_size,
79
+ )
80
+ boundaries = torch.arange(1 / self.num_patches_per_side, 1.0,
81
+ 1 / self.num_patches_per_side)
82
+ position_ids = torch.full(size=(batch_size,
83
+ max_nb_patches_h * max_nb_patches_w),
84
+ fill_value=0)
85
+
86
+ for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
87
+ nb_patches_h = p_attn_mask[:, 0].sum()
88
+ nb_patches_w = p_attn_mask[0].sum()
89
+ fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
90
+ fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
91
+ bucket_coords_h = torch.bucketize(fractional_coords_h,
92
+ boundaries,
93
+ right=True)
94
+ bucket_coords_w = torch.bucketize(fractional_coords_w,
95
+ boundaries,
96
+ right=True)
97
+ pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side +
98
+ bucket_coords_w).flatten()
99
+ position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
100
+ position_ids = position_ids.to(self.position_embedding.weight.device)
101
+ embeddings = embeddings + self.position_embedding(position_ids)
102
+ return embeddings
103
+
104
+
105
+class Idefics2VisionAttention(nn.Module):
106
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
107
+
108
+ def __init__(
109
+ self,
110
+ config: Idefics2Config,
111
+ quant_config: Optional[QuantizationConfig] = None,
112
+ ):
113
+ super().__init__()
114
+ self.config = config
115
+ self.embed_dim = config.hidden_size
116
+ self.num_heads = config.num_attention_heads
117
+ self.head_dim = self.embed_dim // self.num_heads
118
+ if self.head_dim * self.num_heads != self.embed_dim:
119
+ raise ValueError(
120
+ f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" # noqa: E501
121
+ f" {self.num_heads}).")
122
+ self.scale = self.head_dim**-0.5
123
+ self.dropout = config.attention_dropout
124
+ self.qkv_proj = QKVParallelLinear(
125
+ self.embed_dim,
126
+ self.head_dim,
127
+ self.num_heads,
128
+ quant_config=quant_config,
129
+ )
130
+ self.out_proj = RowParallelLinear(
131
+ self.embed_dim,
132
+ self.embed_dim,
133
+ bias=True,
134
+ quant_config=quant_config,
135
+ )
136
+ self.tp_size = get_tensor_model_parallel_world_size()
137
+ self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
138
+ self.is_causal = False
139
+
140
+ def forward(
141
+ self,
142
+ hidden_states: torch.Tensor,
143
+ ) -> torch.Tensor:
144
+ batch_size, q_len, _ = hidden_states.size()
145
+ qkv, _ = self.qkv_proj(
146
+ hidden_states
147
+ ) # batch_size, q_len, 3 * num_heads_per_partition * head_dim
148
+ query_states, key_states, value_states = qkv.chunk(3, dim=-1)
149
+ query_states = query_states.view(batch_size, q_len,
150
+ self.num_heads_per_partition,
151
+ self.head_dim)
152
+ key_states = key_states.view(batch_size, q_len,
153
+ self.num_heads_per_partition,
154
+ self.head_dim)
155
+ value_states = value_states.view(batch_size, q_len,
156
+ self.num_heads_per_partition,
157
+ self.head_dim)
158
+ # see: https://facebookresearch.github.io/xformers/components/ops.html
159
+ out = xops.memory_efficient_attention_forward(
160
+ query_states,
161
+ key_states,
162
+ value_states,
163
+ p=self.dropout,
164
+ scale=self.scale,
165
+ )
166
+ out = out.view(batch_size, q_len, -1)
167
+ attn_output, _ = self.out_proj(out)
168
+ return attn_output
169
+
170
+
171
+class Idefics2VisionMLP(nn.Module):
172
+
173
+ def __init__(
174
+ self,
175
+ config: Idefics2Config,
176
+ quant_config: Optional[QuantizationConfig] = None,
177
+ ):
178
+ super().__init__()
179
+ self.config = config
180
+ self.activation_fn = get_act_fn(config.hidden_act)
181
+ self.fc1 = ColumnParallelLinear(
182
+ config.hidden_size,
183
+ config.intermediate_size,
184
+ bias=True,
185
+ quant_config=quant_config,
186
+ )
187
+ self.fc2 = RowParallelLinear(
188
+ config.intermediate_size,
189
+ config.hidden_size,
190
+ bias=True,
191
+ quant_config=quant_config,
192
+ )
193
+
194
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
195
+ hidden_states, _ = self.fc1(hidden_states)
196
+ hidden_states = self.activation_fn(hidden_states)
197
+ hidden_states, _ = self.fc2(hidden_states)
198
+ return hidden_states
199
+
200
+
201
+class Idefics2EncoderLayer(nn.Module):
202
+
203
+ def __init__(self, config: Idefics2Config):
204
+ super().__init__()
205
+ self.embed_dim = config.hidden_size
206
+ self.self_attn = Idefics2VisionAttention(config)
207
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim,
208
+ eps=config.layer_norm_eps)
209
+ self.mlp = Idefics2VisionMLP(config)
210
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim,
211
+ eps=config.layer_norm_eps)
212
+
213
+ def forward(
214
+ self,
215
+ hidden_states: torch.Tensor,
216
+ ) -> torch.Tensor:
217
+ """
218
+ Args:
219
+ hidden_states (`torch.FloatTensor`):
220
+ Input to the layer of shape `(batch, seq_len, embed_dim)`.
221
+
222
+ """
223
+ residual = hidden_states
224
+ hidden_states = self.layer_norm1(hidden_states)
225
+ hidden_states = self.self_attn(hidden_states)
226
+ hidden_states = residual + hidden_states
227
+ residual = hidden_states
228
+ hidden_states = self.layer_norm2(hidden_states)
229
+ hidden_states = self.mlp(hidden_states)
230
+ hidden_states = residual + hidden_states
231
+ return hidden_states
232
+
233
+
234
+class Idefics2Encoder(nn.Module):
235
+ """
236
+ Transformer encoder consisting of `config.num_hidden_layers` self attention
237
+ layers. Each layer is a
238
+ [`Idefics2EncoderLayer`].
239
+
240
+ Args:
241
+ config: Idefics2Config
242
+ """
243
+
244
+ def __init__(self, config: Idefics2Config):
245
+ super().__init__()
246
+ self.config = config
247
+ self.layers = nn.ModuleList([
248
+ Idefics2EncoderLayer(config)
249
+ for _ in range(config.num_hidden_layers)
250
+ ])
251
+
252
+ def forward(
253
+ self,
254
+ inputs_embeds: torch.Tensor,
255
+ ) -> torch.Tensor:
256
+ r"""
257
+ Args:
258
+ inputs_embeds (torch.Tensor):
259
+ Optionally, instead of passing `input_ids` you can choose to
260
+ directly pass an embedded representation.
261
+ This is useful if you want more control over how to convert
262
+ `input_ids` indices into associated vectorsthan the model's
263
+ internal embedding lookup matrix.
264
+ """
265
+ hidden_states = inputs_embeds
266
+ for encoder_layer in self.layers:
267
+ layer_outputs = encoder_layer(hidden_states)
268
+ hidden_states = layer_outputs
269
+ return hidden_states
270
+
271
+
272
+class Idefics2VisionTransformer(nn.Module):
273
+
274
+ def __init__(self, config: Idefics2VisionConfig):
275
+ super().__init__()
276
+ embed_dim = config.hidden_size
277
+ self.config = config
278
+ self.embeddings = Idefics2VisionEmbeddings(config)
279
+ self.encoder = Idefics2Encoder(config)
280
+ self.post_layernorm = nn.LayerNorm(embed_dim,
281
+ eps=config.layer_norm_eps)
282
+
283
+ def get_input_embeddings(self):
284
+ return self.embeddings
285
+
286
+ def forward(
287
+ self,
288
+ pixel_values,
289
+ patch_attention_mask: Optional[torch.BoolTensor] = None,
290
+ ) -> torch.tensor:
291
+ hidden_states = self.embeddings(
292
+ pixel_values=pixel_values,
293
+ patch_attention_mask=patch_attention_mask)
294
+ encoder_outputs = self.encoder(hidden_states)
295
+ last_hidden_state = self.post_layernorm(encoder_outputs)
296
+ return last_hidden_state
vllm/model_executor/models/minicpmv.py
+639
-384
@@ -24,7 +24,8 @@
24
import math
25
import re
26
from functools import partial
27
-from typing import Dict, Iterable, List, Optional, Tuple, Union
27
+from typing import (Any, Callable, Iterable, List, Optional, Tuple, TypedDict,
28
+ Union)
29
30
import numpy as np
31
import torch
@@ -38,11 +39,14 @@ from transformers.configuration_utils import PretrainedConfig
39
from vllm.attention import AttentionMetadata
40
from vllm.config import CacheConfig, MultiModalConfig
41
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
42
+from vllm.logger import init_logger
43
+from vllm.model_executor.layers.linear import ReplicatedLinear
44
from vllm.model_executor.layers.logits_processor import LogitsProcessor
45
from vllm.model_executor.layers.quantization.base_config import (
46
QuantizationConfig)
47
from vllm.model_executor.layers.sampler import Sampler
48
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
49
+from vllm.model_executor.model_loader.utils import set_default_torch_dtype
50
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51
from vllm.model_executor.models.interfaces import SupportsVision
52
from vllm.model_executor.models.llama import LlamaModel
@@ -54,12 +58,45 @@ from vllm.multimodal.image import (cached_get_image_processor,
58
cached_get_tokenizer)
59
from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
60
61
+from .idefics2_vision_model import Idefics2VisionTransformer
62
+
63
+logger = init_logger(__name__)
64
+
65
_KEYS_TO_MODIFY_MAPPING = {
66
"llm.lm_head": "lm_head",
67
"llm.model": "llm",
68
}
69
70
71
+class MiniCPMVImagePixelInputs(TypedDict):
72
+ pixel_values: List[torch.Tensor]
73
+ """
74
+ Shape: `(batch_size * num_images, num_channels, height, width)`
75
+
76
+ Note that the image size may vary, so we pass it as a list
77
+ instead of a batched tensor.
78
+ """
79
+
80
+ image_bounds: torch.Tensor
81
+ """
82
+ Shape: `(batch_size * num_images, 2)`
83
+
84
+ This should be in `(start, stop)` format.
85
+ """
86
+
87
+ tgt_sizes: torch.Tensor
88
+ """
89
+ Shape: `(batch_size * num_images, 2)`
90
+
91
+ This should be in `(height, width)` format.
92
+ """
93
+
94
+
95
+MiniCPMVImageInputs = MiniCPMVImagePixelInputs
96
+
97
+DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
98
+
99
+
100
def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor):
101
# abs_pos: L, C
102
# tgt_size: (H, W)
@@ -68,23 +105,25 @@ def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor):
105
# tgt_size = int(math.sqrt(tgt_size))
106
dtype = abs_pos.dtype
107
71
- return F.interpolate(
108
+ return (F.interpolate(
109
abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
110
size=(tgt_size[0], tgt_size[1]),
111
mode="bicubic",
112
align_corners=False,
76
- ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)
113
+ ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype))
114
115
116
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
80
-def get_2d_sincos_pos_embed(embed_dim: int,
81
- grid_size: Union[int, Tuple[int, int]],
82
- cls_token: bool = False,
83
- version: Tuple[int, int] = (2, 0)):
117
+def get_2d_sincos_pos_embed(
118
+ embed_dim: int,
119
+ grid_size: Union[int, Tuple[int, int]],
120
+ cls_token: bool = False,
121
+ version: Tuple[int, int] = (2, 0),
122
+):
123
"""
124
grid_size: int of the grid height and width
125
return:
87
- pos_embed: [grid_size*grid_size, embed_dim] or
126
+ pos_embed: [grid_size*grid_size, embed_dim] or
127
[1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
128
"""
129
if isinstance(grid_size, int):
@@ -109,7 +148,7 @@ def get_2d_sincos_pos_embed(embed_dim: int,
148
149
150
def get_2d_sincos_pos_embed_from_grid(embed_dim: int,
112
- grid: Union[int, Tuple[int, int]],
151
+ grid: np.ndarray,
152
version: Tuple[int, int] = (2, 0)):
153
assert embed_dim % 2 == 0
154
@@ -127,7 +166,7 @@ def get_2d_sincos_pos_embed_from_grid(embed_dim: int,
166
167
168
def get_1d_sincos_pos_embed_from_grid(embed_dim: int,
130
- pos: int,
169
+ pos: np.ndarray,
170
version: Tuple[int, int] = (2, 0)):
171
"""
172
embed_dim: output dimension for each position
@@ -136,24 +175,24 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim: int,
175
"""
176
assert embed_dim % 2 == 0
177
omega = np.arange(embed_dim // 2, dtype=np.float32)
139
- omega /= embed_dim / 2.
140
- omega = 1. / 10000**omega # (D/2,)
178
+ omega /= embed_dim / 2.0
179
+ omega = 1.0 / 10000**omega # (D/2,)
180
181
if version == (2, 0):
182
pos = pos.reshape(-1) # (M,)
144
- out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
183
+ out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
184
emb_sin = np.sin(out) # (M, D/2)
185
emb_cos = np.cos(out) # (M, D/2)
186
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
187
else:
149
- out = np.einsum('hw,d->hwd', pos, omega) # (H, W, D/2), outer product
188
+ out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
189
emb_sin = np.sin(out) # (H, W, D/2)
190
emb_cos = np.cos(out) # (H, W, D/2)
191
emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
192
return emb
193
194
156
-class Resampler(nn.Module):
195
+class BaseResampler(nn.Module):
196
"""
197
A 2D perceiver-resampler network with one cross attention layers by
198
(grid_size**2) learnable queries and 2d sincos pos_emb
@@ -161,89 +200,151 @@ class Resampler(nn.Module):
200
A tensor with the shape of (grid_size**2, embed_dim)
201
"""
202
164
- default_norm_layer = partial(nn.LayerNorm, eps=1e-6)
165
-
166
- def __init__(self,
167
- num_queries: int,
168
- grid_size: int,
169
- embed_dim: int,
170
- num_heads: int,
171
- kv_dim: Optional[int] = None,
172
- norm_layer: nn.Module = default_norm_layer,
173
- adaptive: bool = False,
174
- max_size: Tuple[int, int] = (70, 70),
175
- version: Tuple[int, int] = (2, 0)):
203
+ def __init__(
204
+ self,
205
+ num_queries: int,
206
+ embed_dim: int,
207
+ num_heads: int,
208
+ kv_dim: Optional[int] = None,
209
+ norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
210
+ ) -> None:
211
super().__init__()
212
178
- self.version = version
179
- if self.version == (2, 0):
180
- self.num_queries = grid_size**2
181
- else:
182
- self.num_queries = num_queries
183
- self.max_size = max_size
213
+ self.num_queries = num_queries
214
self.embed_dim = embed_dim
215
self.num_heads = num_heads
186
- self.adaptive = adaptive
216
217
self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
189
- trunc_normal_(self.query, std=.02)
218
+ trunc_normal_(self.query, std=0.02)
219
220
if kv_dim is not None and kv_dim != embed_dim:
192
- self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
221
+ self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
222
else:
194
- self.kv_proj = nn.Identity()
223
+ # Maintain the same return value with ReplicatedLinear.forward
224
+ self.kv_proj = lambda *args, **kwargs: (
225
+ nn.Identity()(*args, **kwargs),
226
+ None,
227
+ )
228
229
self.attn = nn.MultiheadAttention(embed_dim, num_heads)
230
self.ln_q = norm_layer(embed_dim)
231
self.ln_kv = norm_layer(embed_dim)
199
-
232
self.ln_post = norm_layer(embed_dim)
233
self.proj = nn.Parameter(
234
(embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
235
204
- if self.version == (2, 0):
205
- self.pos_embed = nn.Parameter(
206
- torch.from_numpy(
207
- get_2d_sincos_pos_embed(
208
- embed_dim, grid_size,
209
- version=self.version)).float()).requires_grad_(False)
236
+ def _init_weights(self, m: nn.Module) -> None:
237
+ if isinstance(m, nn.Linear):
238
+ trunc_normal_(m.weight, std=0.02)
239
+ if isinstance(m, nn.Linear) and m.bias is not None:
240
+ nn.init.constant_(m.bias, 0)
241
+ elif isinstance(m, nn.LayerNorm):
242
+ nn.init.constant_(m.bias, 0)
243
+ nn.init.constant_(m.weight, 1.0)
244
+
245
+ def _repeat(self, query, N: int):
246
+ return query.unsqueeze(1).repeat(1, N, 1)
247
+
248
+
249
+class Resampler2(BaseResampler):
250
+
251
+ def __init__(
252
+ self,
253
+ grid_size: int,
254
+ embed_dim: int,
255
+ num_heads: int,
256
+ kv_dim: Optional[int] = None,
257
+ norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
258
+ adaptive: bool = False,
259
+ ) -> None:
260
+ super().__init__(grid_size**2, embed_dim, num_heads, kv_dim,
261
+ norm_layer)
262
+
263
+ self.adaptive = adaptive
264
+
265
+ pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
266
+ grid_size,
267
+ version=(2, 0))
268
+ self.pos_embed = nn.Parameter(
269
+ torch.from_numpy(pos_embed_arr).float()).requires_grad_(False)
270
+
271
+ self.apply(self._init_weights)
272
+
273
+ def forward(
274
+ self,
275
+ x: torch.Tensor,
276
+ tgt_sizes: torch.Tensor,
277
+ attn_mask: Optional[torch.Tensor] = None,
278
+ ):
279
+ if self.adaptive:
280
+ pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
281
+ tgt_sizes,
282
+ version=(2, 0))
283
+ pos_embed = torch.from_numpy(pos_embed_arr).to(device=x.device,
284
+ dtype=x.dtype)
285
else:
211
- self._set_2d_pos_cache(self.max_size)
286
+ pos_embed = get_abs_pos(self.pos_embed, tgt_sizes)
287
+
288
+ x, _ = self.kv_proj(x)
289
+ x = self.ln_kv(x).permute(1, 0, 2)
290
+
291
+ N = x.shape[1]
292
+ q = self.ln_q(self.query)
293
+ out = self.attn(
294
+ self._repeat(q, N) + self.pos_embed.unsqueeze(1),
295
+ x + pos_embed.unsqueeze(1),
296
+ x,
297
+ attn_mask=attn_mask,
298
+ )[0]
299
+ x = out.permute(1, 0, 2)
300
+
301
+ x = self.ln_post(x)
302
+ x = x @ self.proj
303
+ return x
304
+
305
+
306
+class Resampler2_5(BaseResampler):
307
+
308
+ def __init__(
309
+ self,
310
+ num_queries: int,
311
+ embed_dim: int,
312
+ num_heads: int,
313
+ kv_dim: Optional[int] = None,
314
+ norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
315
+ max_size: Tuple[int, int] = (70, 70),
316
+ ) -> None:
317
+ super().__init__(num_queries, embed_dim, num_heads, kv_dim, norm_layer)
318
+
319
+ self.max_size = max_size
320
+ self._set_2d_pos_cache(self.max_size)
321
322
self.apply(self._init_weights)
323
324
def _set_2d_pos_cache(self,
325
max_size: Tuple[int, int],
217
- device: torch.types.Device = 'cpu'):
218
- pos_embed = torch.from_numpy(
219
- get_2d_sincos_pos_embed(self.embed_dim,
220
- max_size,
221
- version=self.version)).float().to(device)
326
+ device: torch.types.Device = "cpu") -> None:
327
+ pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
328
+ max_size,
329
+ version=(2, 5))
330
+ pos_embed = torch.from_numpy(pos_embed_arr).float().to(device)
331
self.register_buffer("pos_embed", pos_embed, persistent=False)
332
333
def _adjust_pos_cache(self, tgt_sizes: torch.Tensor,
225
- device: torch.types.Device):
226
- max_h = torch.max(tgt_sizes[:, 0])
227
- max_w = torch.max(tgt_sizes[:, 1])
334
+ device: torch.types.Device) -> None:
335
+ max_h = tgt_sizes[:, 0].max().item()
336
+ max_w = tgt_sizes[:, 1].max().item()
337
+ assert isinstance(max_h, int) and isinstance(max_w, int)
338
+
339
if max_h > self.max_size[0] or max_w > self.max_size[1]:
229
- self.max_size = [
340
+ self.max_size = (
341
max(max_h, self.max_size[0]),
231
- max(max_w, self.max_size[1])
232
- ]
342
+ max(max_w, self.max_size[1]),
343
+ )
344
self._set_2d_pos_cache(self.max_size, device)
345
235
- def _init_weights(self, m: nn.Module):
236
- if isinstance(m, nn.Linear):
237
- trunc_normal_(m.weight, std=.02)
238
- if isinstance(m, nn.Linear) and m.bias is not None:
239
- nn.init.constant_(m.bias, 0)
240
- elif isinstance(m, nn.LayerNorm):
241
- nn.init.constant_(m.bias, 0)
242
- nn.init.constant_(m.weight, 1.0)
243
-
244
- def forward_2_5(self,
245
- x: torch.Tensor,
246
- tgt_sizes: Optional[torch.Tensor] = None):
346
+ def forward(self, x: torch.Tensor,
347
+ tgt_sizes: torch.Tensor) -> torch.Tensor:
348
assert x.shape[0] == tgt_sizes.shape[0]
349
bs = x.shape[0]
350
@@ -254,25 +355,25 @@ class Resampler(nn.Module):
355
356
self._adjust_pos_cache(tgt_sizes, device=device)
357
257
- max_patch_len = torch.max(patch_len)
358
+ max_patch_len = patch_len.max().item()
359
+ assert isinstance(max_patch_len, int)
360
+
361
key_padding_mask = torch.zeros((bs, max_patch_len),
362
dtype=torch.bool,
363
device=device)
364
365
pos_embed = []
366
for i in range(bs):
264
- tgt_h, tgt_w = tgt_sizes[i]
367
+ tgt_h, tgt_w = tgt_sizes[i].tolist()
368
pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape(
369
(tgt_h * tgt_w, -1)).to(dtype)) # patches * D
370
key_padding_mask[i, patch_len[i]:] = True
268
-
371
pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed,
372
batch_first=True,
373
padding_value=0.0).permute(
374
1, 0,
375
2) # BLD => L * B * D
274
-
275
- x = self.kv_proj(x) # B * L * D
376
+ x, _ = self.kv_proj(x) # B * L * D
377
x = self.ln_kv(x).permute(1, 0, 2) # L * B * D
378
379
q = self.ln_q(self.query) # Q * D
@@ -281,7 +382,8 @@ class Resampler(nn.Module):
382
self._repeat(q, bs), # Q * B * D
383
x + pos_embed, # L * B * D + L * B * D
384
x,
284
- key_padding_mask=key_padding_mask)[0]
385
+ key_padding_mask=key_padding_mask,
386
+ )[0]
387
# out: Q * B * D
388
x = out.permute(1, 0, 2) # B * Q * D
389
@@ -289,45 +391,6 @@ class Resampler(nn.Module):
391
x = x @ self.proj
392
return x
393
292
- def forward_2(self,
293
- x: torch.Tensor,
294
- tgt_sizes: Optional[torch.Tensor] = None,
295
- attn_mask: Optional[torch.Tensor] = None):
296
- if self.adaptive:
297
- pos_embed = torch.Tensor(
298
- get_2d_sincos_pos_embed(self.embed_dim,
299
- tgt_sizes)).float().to(device=x.device,
300
- dtype=x.dtype)
301
- else:
302
- pos_embed = get_abs_pos(self.pos_embed, tgt_sizes)
303
-
304
- x = self.kv_proj(x)
305
- x = self.ln_kv(x).permute(1, 0, 2)
306
-
307
- N = x.shape[1]
308
- q = self.ln_q(self.query)
309
- out = self.attn(self._repeat(q, N) + self.pos_embed.unsqueeze(1),
310
- x + pos_embed.unsqueeze(1),
311
- x,
312
- attn_mask=attn_mask)[0]
313
- x = out.permute(1, 0, 2)
314
-
315
- x = self.ln_post(x)
316
- x = x @ self.proj
317
- return x
318
-
319
- def forward(self,
320
- x: torch.Tensor,
321
- tgt_sizes: Optional[torch.Tensor] = None,
322
- attn_mask: Optional[torch.Tensor] = None):
323
- if self.version == (2, 0):
324
- return self.forward_2(x, tgt_sizes=tgt_sizes, attn_mask=attn_mask)
325
- else:
326
- return self.forward_2_5(x, tgt_sizes=tgt_sizes)
327
-
328
- def _repeat(self, query, N: int):
329
- return query.unsqueeze(1).repeat(1, N, 1)
330
-
394
395
def get_max_minicpmv_image_tokens(ctx: InputContext):
396
hf_config = ctx.get_hf_config(PretrainedConfig)
@@ -348,10 +411,7 @@ def dummy_image_for_minicpmv(hf_config: PretrainedConfig):
411
def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int):
412
hf_config = ctx.get_hf_config(PretrainedConfig)
413
351
- # image_feature_size = get_max_minicpmv_image_tokens(ctx)
352
-
414
seq_data = dummy_seq_data_for_minicpmv(seq_len)
354
-
415
mm_data = dummy_image_for_minicpmv(hf_config)
416
417
return seq_data, mm_data
@@ -376,25 +436,36 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
436
pattern = "(<image>./</image>)"
437
image = multi_modal_data["image"]
438
image_tags = re.findall(pattern, prompt)
379
- assert len(image_tags) <= 1
380
- text_chunks = prompt.split(pattern)
381
- new_prompt = text_chunks[0] \
382
- + image_processor.get_slice_image_placeholder(image.size) \
383
- + text_chunks[1]
439
385
- new_token_ids = tokenizer.encode(new_prompt)
386
-
387
- llm_inputs = LLMInputs(prompt_token_ids=new_token_ids,
388
- prompt=new_prompt,
389
- multi_modal_data=multi_modal_data)
440
+ if len(image_tags) == 0:
441
+ new_token_ids = token_ids
442
+ new_prompt = prompt
443
+ else:
444
+ if len(image_tags) > 1:
445
+ logger.warning("Multiple image input is not supported yet, "
446
+ "so any extra image tokens will be treated "
447
+ "as plain text.")
448
+
449
+ text_chunks = prompt.split(pattern)
450
+ new_prompt = (text_chunks[0] +
451
+ image_processor.get_slice_image_placeholder(image.size) +
452
+ "".join(text_chunks[1:]))
453
+
454
+ new_token_ids = tokenizer.encode(new_prompt)
455
+
456
+ llm_inputs = LLMInputs(
457
+ prompt_token_ids=new_token_ids,
458
+ prompt=new_prompt,
459
+ multi_modal_data=multi_modal_data,
460
+ )
461
return llm_inputs
462
463
393
-@MULTIMODAL_REGISTRY.register_image_input_mapper()
394
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
395
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
396
-@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
397
-class MiniCPMV(nn.Module, SupportsVision):
464
+class MiniCPMVBaseModel(nn.Module, SupportsVision):
465
+ """
466
+ The abstract class of MiniCPMV can only be inherited, but cannot be
467
+ instantiated.
468
+ """
469
470
def __init__(
471
self,
@@ -419,8 +490,8 @@ class MiniCPMV(nn.Module, SupportsVision):
490
self.vpm = self.init_vision_module()
491
param_dtype = torch.get_default_dtype()
492
self.vpm.to(dtype=param_dtype)
422
- self.vision_dim = self.vpm.embed_dim if self.version == (2, 0) \
423
- else self.vpm.embeddings.embed_dim
493
+ self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
494
+ self.vpm.embeddings.embed_dim)
495
self.embed_dim = self.config.hidden_size
496
self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
497
self.resampler.to(device="cuda", dtype=param_dtype)
@@ -430,248 +501,100 @@ class MiniCPMV(nn.Module, SupportsVision):
501
self.logits_processor = LogitsProcessor(config.vocab_size)
502
self.sampler = Sampler()
503
433
- def init_llm(self,
434
- config: PretrainedConfig,
435
- cache_config: Optional[CacheConfig] = None,
436
- quant_config: Optional[QuantizationConfig] = None):
437
- if self.version == (2, 0):
438
- return MiniCPMModel(config,
439
- cache_config=cache_config,
440
- quant_config=quant_config)
441
- elif self.version == (2, 5):
442
- return LlamaModel(config,
443
- cache_config=cache_config,
444
- quant_config=quant_config)
445
- else:
446
- return Qwen2Model(config,
447
- cache_config=cache_config,
448
- quant_config=quant_config)
449
-
450
- def init_vision_module(self):
451
- if self.version == (2, 0):
452
- try:
453
- import timm
454
- except ImportError:
455
- raise ImportError(
456
- 'Please install timm==0.9.10') from ImportError
457
- default_dtype = torch.get_default_dtype()
458
- torch.set_default_dtype(torch.float16)
459
- model = timm.create_model('vit_so400m_patch14_siglip_384.webli',
460
- pretrained=False,
461
- num_classes=0,
462
- dynamic_img_size=True,
463
- dynamic_img_pad=True)
464
- torch.set_default_dtype(default_dtype)
465
- if isinstance(model, timm.models.VisionTransformer
466
- ) and model.attn_pool is not None:
467
- model.attn_pool = torch.nn.Identity()
468
-
469
- if self.config.drop_vision_last_layer:
470
- model.blocks = model.blocks[:-1]
471
- elif self.version == (2, 5):
472
- from transformers.models.idefics2.modeling_idefics2 import (
473
- Idefics2VisionTransformer)
474
- model = Idefics2VisionTransformer(self.config.vision_config)
475
- if self.config.drop_vision_last_layer:
476
- model.encoder.layers = model.encoder.layers[:-1]
477
- else:
478
- from vllm.model_executor.models.na_vit import (
479
- SiglipVisionTransformer)
480
- if self.config._attn_implementation == 'flash_attention_2':
481
- self.config.vision_config._attn_implementation \
482
- = 'flash_attention_2'
483
- else:
484
- # not support sdpa
485
- self.config.vision_config._attn_implementation = 'eager'
486
- model = SiglipVisionTransformer(self.config.vision_config)
487
- if self.config.drop_vision_last_layer:
488
- model.encoder.layers = model.encoder.layers[:-1]
489
- return model
490
-
491
- def init_resampler(self, embed_dim: int, vision_dim: int):
492
- default_dtype = torch.get_default_dtype()
493
- torch.set_default_dtype(torch.float16)
494
- if self.version == (2, 0):
495
- resampler = Resampler(grid_size=int(
496
- math.sqrt(self.config.query_num)),
497
- num_queries=None,
498
- embed_dim=embed_dim,
499
- num_heads=embed_dim // 128,
500
- kv_dim=vision_dim,
501
- adaptive=True,
502
- version=self.version)
504
+ def get_embedding(
505
+ self,
506
+ input_ids: torch.Tensor,
507
+ image_inputs: Optional[MiniCPMVImageInputs],
508
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
509
+ vlm_embedding: torch.Tensor = self.llm.embed_tokens(input_ids)
510
+ if hasattr(self.config, "scale_emb"):
511
+ vlm_embedding *= self.config.scale_emb
512
+
513
+ if image_inputs is None: # No image
514
+ vision_hidden_states = torch.tensor([], device=input_ids.device)
515
else:
504
- resampler = Resampler(num_queries=self.config.query_num,
505
- grid_size=None,
506
- embed_dim=embed_dim,
507
- num_heads=embed_dim // 128,
508
- kv_dim=vision_dim,
509
- adaptive=True,
510
- version=self.version)
511
- torch.set_default_dtype(default_dtype)
512
- return resampler
516
+ vision_hidden_states = self.get_vision_hidden_states(image_inputs)
517
+
518
+ # See NOTE in _parse_and_validate_inputs
519
+ image_bounds = image_inputs["image_bounds"]
520
+ if len(image_bounds) > 0:
521
+ image_indices = torch.stack([
522
+ torch.arange(start, end, dtype=torch.long)
523
+ for start, end in image_bounds.tolist()
524
+ ]).to(vlm_embedding.device)
525
+ vlm_embedding.scatter_(
526
+ 0,
527
+ image_indices.view(-1, 1).repeat(1,
528
+ vlm_embedding.shape[-1]),
529
+ vision_hidden_states.view(-1,
530
+ vision_hidden_states.shape[-1]),
531
+ )
532
514
- def get_vision_embedding(self,
515
- pixel_values: List[List[torch.Tensor]],
516
- patch_attn_mask: Optional[torch.Tensor] = None,
517
- tgt_sizes: Optional[torch.Tensor] = None,
518
- version: Tuple[int, int] = (2, 0)):
519
- if version == (2, 0):
520
- res = []
521
- dtype = self.vpm.pos_embed.data.dtype
522
- for pixel_value in pixel_values:
523
- # V2.0 start
524
- H, W = pixel_value[0].shape[-2:]
525
- tgt_size = (math.ceil(H / self.vpm.patch_embed.patch_size[0]),
526
- math.ceil(W / self.vpm.patch_embed.patch_size[0]))
527
- # V2.0 end
528
- vision_embedding = self.vpm.forward_features(
529
- pixel_value.unsqueeze(0).type(dtype))
530
- if hasattr(self.vpm, 'num_prefix_tokens'
531
- ) and self.vpm.num_prefix_tokens > 0:
532
- vision_embedding = vision_embedding[:, self.vpm.
533
- num_prefix_tokens:]
534
- res.append(self.resampler(vision_embedding, tgt_size))
535
- return torch.vstack(res)
536
- elif version == (2, 5):
537
- vision_embedding = self.vpm(
538
- pixel_values.type(dtype),
539
- patch_attention_mask=patch_attn_mask).last_hidden_state
540
- vision_embedding = self.resampler(vision_embedding, tgt_sizes)
541
- else:
542
- vision_embedding = self.vpm(pixel_values.type(dtype),
543
- patch_attention_mask=patch_attn_mask,
544
- tgt_sizes=tgt_sizes).last_hidden_state
533
+ return vlm_embedding, vision_hidden_states
534
546
- def get_image_bounds(self, input_ids: torch.Tensor):
535
+ def _get_image_bounds(self, input_ids: torch.Tensor) -> torch.Tensor:
536
tokenizer = cached_get_tokenizer(self.config._name_or_path,
537
trust_remote_code=True)
549
- if not hasattr(tokenizer, "slice_start_id"):
550
- start_cond = input_ids == tokenizer.im_start_id
551
- end_cond = input_ids == tokenizer.im_end_id
552
- else:
553
- start_cond = (input_ids == tokenizer.im_start_id) | (
554
- input_ids == tokenizer.slice_start_id)
555
- end_cond = (input_ids == tokenizer.im_end_id) | (
556
- input_ids == tokenizer.slice_end_id)
538
+ start_cond = input_ids == tokenizer.im_start_id
539
+ end_cond = input_ids == tokenizer.im_end_id
540
+ if hasattr(tokenizer, "slice_start_id"):
541
+ start_cond |= (input_ids == tokenizer.slice_start_id)
542
+ end_cond |= (input_ids == tokenizer.slice_end_id)
543
558
- image_start_tokens = torch.where(start_cond)[0]
544
+ image_start_tokens, = torch.where(start_cond)
545
image_start_tokens += 1
560
- image_end_tokens = torch.where(end_cond)[0]
546
+ image_end_tokens, = torch.where(end_cond)
547
valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))
548
+
549
if valid_image_nums == 0:
563
- return []
564
- image_bound = torch.hstack([
550
+ return torch.zeros((0, 2), device=input_ids.device)
551
+
552
+ return torch.hstack([
553
image_start_tokens[:valid_image_nums].unsqueeze(-1),
554
image_end_tokens[:valid_image_nums].unsqueeze(-1),
555
])
556
569
- return image_bound
570
-
571
- def get_vision_hidden_states(self, data: Dict[str,
572
- Union[List[torch.Tensor],
573
- torch.Tensor]]):
574
- if "vision_hidden_states" not in data:
575
- pixel_values = data["pixel_values"]
576
- tgt_sizes = data["tgt_sizes"]
577
- vision_hidden_states = []
578
- if self.version == (2, 0):
579
- if pixel_values is not None and len(pixel_values) > 0:
580
- vision_hidden_states = self.get_vision_embedding(
581
- pixel_values)
582
- else:
583
- vision_hidden_states = torch.tensor([]).to(
584
- data["input_ids"].device)
585
- else:
586
- device = self.vpm.embeddings.position_embedding.weight.device
587
- dtype = self.vpm.embeddings.position_embedding.weight.dtype
588
- all_pixel_values = [
589
- i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
590
- ]
591
- if all_pixel_values:
592
- tgt_sizes = torch.vstack(tgt_sizes).type(torch.int32)
593
- max_patches = torch.max(tgt_sizes[:, 0] * tgt_sizes[:, 1])
594
- all_pixel_values = torch.nn.utils.rnn.pad_sequence(
595
- all_pixel_values, batch_first=True, padding_value=0.0)
596
- B, L, _ = all_pixel_values.shape
597
- all_pixel_values = all_pixel_values.permute(
598
- 0, 2, 1).reshape(B, 3, -1, L)
599
- patch_attn_mask = torch.zeros((B, 1, max_patches),
600
- dtype=torch.bool,
601
- device=device)
602
- if self.version == (2, 5):
603
- for i in range(B):
604
- patch_attn_mask[i, :tgt_sizes[i][0] *
605
- tgt_sizes[i][1]] = True
606
- vision_embedding = self.vpm(
607
- all_pixel_values.type(dtype),
608
- patch_attention_mask=patch_attn_mask
609
- ).last_hidden_state
610
- else:
611
- for i in range(B):
612
- patch_attn_mask[i, 0, :tgt_sizes[i][0] *
613
- tgt_sizes[i][1]] = True
614
- vision_embedding = self.vpm(
615
- all_pixel_values.type(dtype),
616
- patch_attention_mask=patch_attn_mask,
617
- tgt_sizes=tgt_sizes).last_hidden_state
618
-
619
- vision_hidden_states = self.resampler(
620
- vision_embedding, tgt_sizes)
621
-
622
- else: # no image
623
- dummy_feature = []
624
- vision_hidden_states = dummy_feature
625
- else:
626
- vision_hidden_states = data["vision_hidden_states"]
627
-
628
- return vision_hidden_states
629
-
630
- def get_embedding(self, data: Dict[str, Union[List[torch.Tensor],
631
- torch.Tensor]]):
632
- input_ids = data["input_ids"]
633
-
634
- vision_hidden_states = self.get_vision_hidden_states(data)
635
- if vision_hidden_states is not None and len(vision_hidden_states) > 0:
636
- image_bounds = self.get_image_bounds(input_ids)
637
- else:
638
- image_bounds = []
639
-
640
- if hasattr(self.config, 'scale_emb'):
641
- vlm_embedding = self.llm.embed_tokens(
642
- input_ids) * self.config.scale_emb
643
- else:
644
- vlm_embedding = self.llm.embed_tokens(input_ids)
645
- vision_hidden_states = [
646
- i.type(vlm_embedding.dtype) if isinstance(i, torch.Tensor) else i
647
- for i in vision_hidden_states
648
- ]
649
-
650
- if len(vision_hidden_states) > 0 and len(image_bounds) > 0:
651
- vision_hidden_states = torch.cat(vision_hidden_states, dim=0)
652
- image_indices = torch.stack([
653
- torch.arange(r[0], r[1], dtype=torch.long)
654
- for r in image_bounds
655
- ]).to(vlm_embedding.device)
656
- vlm_embedding.scatter_(
657
- 0,
658
- image_indices.view(-1, 1).repeat(1, vlm_embedding.shape[-1]),
659
- vision_hidden_states.view(-1, vision_hidden_states.shape[-1]))
660
- return vlm_embedding, vision_hidden_states
661
-
662
- def process_multimodal_inputs(self, inputs: Dict[str,
663
- Union[List[torch.Tensor],
664
- torch.Tensor]]):
665
- pixel_values = []
666
- tgt_sizes = []
667
- for b in range(len(inputs["pixel_values"])):
668
- pixel_values += inputs["pixel_values"][b]
669
- tgt_sizes += inputs["tgt_sizes"][b]
670
- return {
671
- "pixel_values": pixel_values,
672
- "input_ids": inputs["input_ids"],
673
- "tgt_sizes": tgt_sizes
674
- }
557
+ def _parse_and_validate_inputs(
558
+ self,
559
+ input_ids: torch.Tensor,
560
+ **kwargs: object,
561
+ ) -> Optional[MiniCPMVImageInputs]:
562
+ pixel_values = kwargs.pop("pixel_values", [])
563
+ tgt_sizes = kwargs.pop("tgt_sizes", [])
564
+
565
+ if not isinstance(pixel_values, (torch.Tensor, list)):
566
+ raise ValueError("Incorrect type of pixel values. "
567
+ f"Got type: {type(pixel_values)}")
568
+
569
+ if not isinstance(tgt_sizes, (torch.Tensor, list)):
570
+ raise ValueError("Incorrect type of target sizes. "
571
+ f"Got type: {type(tgt_sizes)}")
572
+
573
+ if len(pixel_values) != len(tgt_sizes):
574
+ raise ValueError("Inconsistent batch lengths, found: "
575
+ f"{len(pixel_values)} vs. {len(tgt_sizes)}")
576
+
577
+ pixel_values_flat: List[torch.Tensor] = []
578
+ tgt_sizes_flat: List[torch.Tensor] = []
579
+ for b in range(len(pixel_values)):
580
+ pixel_values_flat += pixel_values[b]
581
+ tgt_sizes_flat += tgt_sizes[b]
582
+
583
+ # NOTE: Input IDs does not contain image tokens during memory profiling,
584
+ # so we allow it to be empty
585
+ if len(pixel_values_flat) != len(tgt_sizes_flat):
586
+ raise ValueError("Inconsistent flattened lengths, found: "
587
+ f"{len(pixel_values_flat)} vs. "
588
+ f"{len(tgt_sizes_flat)}")
589
+
590
+ if len(pixel_values_flat) == 0:
591
+ return None
592
+
593
+ return MiniCPMVImageInputs(
594
+ image_bounds=self._get_image_bounds(input_ids),
595
+ pixel_values=pixel_values_flat,
596
+ tgt_sizes=torch.stack(tgt_sizes_flat),
597
+ )
598
599
def forward(
600
self,
@@ -680,23 +603,20 @@ class MiniCPMV(nn.Module, SupportsVision):
603
kv_caches: List[torch.Tensor],
604
attn_metadata: AttentionMetadata,
605
intermediate_tensors: Optional[IntermediateTensors] = None,
683
- **kwargs: object,
684
- ):
685
- inputs = {
686
- "pixel_values": kwargs.pop("pixel_values", []),
687
- "input_ids": input_ids,
688
- "tgt_sizes": kwargs.pop("tgt_sizes", None),
689
- }
690
- inputs = self.process_multimodal_inputs(inputs)
691
-
692
- vlm_embeddings, vision_hidden_states = self.get_embedding(inputs)
693
-
694
- output = self.llm(input_ids=None,
695
- positions=positions,
696
- kv_caches=kv_caches,
697
- attn_metadata=attn_metadata,
698
- intermediate_tensors=intermediate_tensors,
699
- inputs_embeds=vlm_embeddings)
606
+ **kwargs: Any,
607
+ ) -> torch.Tensor:
608
+ image_inputs = self._parse_and_validate_inputs(input_ids, **kwargs)
609
+
610
+ vlm_embeddings, _ = self.get_embedding(input_ids, image_inputs)
611
+
612
+ output = self.llm(
613
+ input_ids=None,
614
+ positions=positions,
615
+ kv_caches=kv_caches,
616
+ attn_metadata=attn_metadata,
617
+ intermediate_tensors=intermediate_tensors,
618
+ inputs_embeds=vlm_embeddings,
619
+ )
620
return output
621
622
def compute_logits(self, hidden_states: torch.Tensor,
@@ -735,13 +655,10 @@ class MiniCPMV(nn.Module, SupportsVision):
655
# the checkpoint. Skip them.
656
continue
657
use_default_weight_loading = False
738
- if "vpm" in name or 'resampler' in name:
739
- # We only do sharding for language model and
740
- # not vision model for now.
658
+ if self.is_default_weight_loading(name):
659
use_default_weight_loading = True
660
else:
743
- for (param_name, weight_name,
744
- shard_id) in stacked_params_mapping:
661
+ for param_name, weight_name, shard_id in stacked_params_mapping:
662
if weight_name not in name:
663
continue
664
param = params_dict[name.replace(weight_name, param_name)]
@@ -755,3 +672,341 @@ class MiniCPMV(nn.Module, SupportsVision):
672
weight_loader = getattr(param, "weight_loader",
673
default_weight_loader)
674
weight_loader(param, loaded_weight)
675
+
676
+ def init_llm(
677
+ self,
678
+ config: PretrainedConfig,
679
+ cache_config: Optional[CacheConfig] = None,
680
+ quant_config: Optional[QuantizationConfig] = None,
681
+ ) -> nn.Module:
682
+ raise NotImplementedError
683
+
684
+ def init_vision_module(self) -> nn.Module:
685
+ raise NotImplementedError
686
+
687
+ def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
688
+ raise NotImplementedError
689
+
690
+ def get_vision_embedding(
691
+ self,
692
+ pixel_values: List[torch.Tensor],
693
+ patch_attn_mask: Optional[torch.Tensor] = None,
694
+ tgt_sizes: Optional[torch.Tensor] = None,
695
+ ) -> torch.Tensor:
696
+ raise NotImplementedError
697
+
698
+ def get_vision_hidden_states(self,
699
+ data: MiniCPMVImageInputs) -> torch.Tensor:
700
+ raise NotImplementedError
701
+
702
+ def is_default_weight_loading(self, name: str) -> bool:
703
+ raise NotImplementedError
704
+
705
+
706
+class MiniCPMV2(MiniCPMVBaseModel):
707
+
708
+ def __init__(
709
+ self,
710
+ config: PretrainedConfig,
711
+ multimodal_config: MultiModalConfig,
712
+ cache_config: Optional[CacheConfig] = None,
713
+ quant_config: Optional[QuantizationConfig] = None,
714
+ ):
715
+ super().__init__(config, multimodal_config, cache_config, quant_config)
716
+ assert self.version == (2, 0)
717
+
718
+ def init_llm(
719
+ self,
720
+ config: PretrainedConfig,
721
+ cache_config: Optional[CacheConfig] = None,
722
+ quant_config: Optional[QuantizationConfig] = None,
723
+ ) -> nn.Module:
724
+ return MiniCPMModel(config,
725
+ cache_config=cache_config,
726
+ quant_config=quant_config)
727
+
728
+ def init_vision_module(self) -> nn.Module:
729
+ # TODO :refactor this vision model
730
+ try:
731
+ import timm
732
+ except ImportError:
733
+ raise ImportError("Please install timm==0.9.10") from ImportError
734
+ with set_default_torch_dtype(torch.float16):
735
+ model = timm.create_model(
736
+ "vit_so400m_patch14_siglip_384.webli",
737
+ pretrained=False,
738
+ num_classes=0,
739
+ dynamic_img_size=True,
740
+ dynamic_img_pad=True,
741
+ )
742
+
743
+ if (isinstance(model, timm.models.VisionTransformer)
744
+ and model.attn_pool is not None):
745
+ model.attn_pool = torch.nn.Identity()
746
+
747
+ if self.config.drop_vision_last_layer:
748
+ model.blocks = model.blocks[:-1]
749
+
750
+ return model
751
+
752
+ def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
753
+ with set_default_torch_dtype(torch.float16):
754
+ resampler = Resampler2(
755
+ embed_dim=embed_dim,
756
+ num_heads=embed_dim // 128,
757
+ grid_size=int(math.sqrt(self.config.query_num)),
758
+ kv_dim=vision_dim,
759
+ adaptive=True,
760
+ )
761
+
762
+ return resampler
763
+
764
+ def get_vision_embedding(
765
+ self,
766
+ pixel_values: List[torch.Tensor],
767
+ patch_attn_mask: Optional[torch.Tensor] = None,
768
+ tgt_sizes: Optional[torch.Tensor] = None,
769
+ ) -> torch.Tensor:
770
+ res = []
771
+ dtype = self.vpm.pos_embed.data.dtype
772
+ for pixel_value in pixel_values:
773
+ H, W = pixel_value[0].shape[-2:]
774
+ tgt_size = (
775
+ math.ceil(H / self.vpm.patch_embed.patch_size[0]),
776
+ math.ceil(W / self.vpm.patch_embed.patch_size[0]),
777
+ )
778
+ vision_embedding = self.vpm.forward_features(
779
+ pixel_value.unsqueeze(0).type(dtype))
780
+ if (hasattr(self.vpm, "num_prefix_tokens")
781
+ and self.vpm.num_prefix_tokens > 0):
782
+ vision_embedding = vision_embedding[:, self.vpm.
783
+ num_prefix_tokens:]
784
+ res.append(self.resampler(vision_embedding, tgt_size))
785
+ return torch.vstack(res)
786
+
787
+ def get_vision_hidden_states(self,
788
+ data: MiniCPMVImageInputs) -> torch.Tensor:
789
+ pixel_values = data["pixel_values"]
790
+
791
+ return self.get_vision_embedding(pixel_values)
792
+
793
+ def is_default_weight_loading(self, name: str) -> bool:
794
+ return "resampler" in name or "vpm" in name
795
+
796
+
797
+class MiniCPMV2_5(MiniCPMVBaseModel):
798
+
799
+ def __init__(
800
+ self,
801
+ config: PretrainedConfig,
802
+ multimodal_config: MultiModalConfig,
803
+ cache_config: Optional[CacheConfig] = None,
804
+ quant_config: Optional[QuantizationConfig] = None,
805
+ ):
806
+ super().__init__(config, multimodal_config, cache_config, quant_config)
807
+ assert self.version == (2, 5)
808
+
809
+ def init_llm(
810
+ self,
811
+ config: PretrainedConfig,
812
+ cache_config: Optional[CacheConfig] = None,
813
+ quant_config: Optional[QuantizationConfig] = None,
814
+ ) -> nn.Module:
815
+ return LlamaModel(config,
816
+ cache_config=cache_config,
817
+ quant_config=quant_config)
818
+
819
+ def init_vision_module(self) -> nn.Module:
820
+ model = Idefics2VisionTransformer(self.config.vision_config)
821
+ if self.config.drop_vision_last_layer:
822
+ model.encoder.layers = model.encoder.layers[:-1]
823
+ return model
824
+
825
+ def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
826
+ with set_default_torch_dtype(torch.float16):
827
+ resampler = Resampler2_5(
828
+ num_queries=self.config.query_num,
829
+ embed_dim=embed_dim,
830
+ num_heads=embed_dim // 128,
831
+ kv_dim=vision_dim,
832
+ )
833
+ return resampler
834
+
835
+ def get_vision_embedding(
836
+ self,
837
+ pixel_values: List[torch.Tensor],
838
+ patch_attn_mask: Optional[torch.Tensor] = None,
839
+ tgt_sizes: Optional[torch.Tensor] = None,
840
+ ) -> torch.Tensor:
841
+ vision_embedding = self.vpm(pixel_values,
842
+ patch_attention_mask=patch_attn_mask)
843
+ vision_embedding = self.resampler(vision_embedding, tgt_sizes)
844
+ return vision_embedding
845
+
846
+ def get_vision_hidden_states(self,
847
+ data: MiniCPMVImageInputs) -> torch.Tensor:
848
+ pixel_values = data["pixel_values"]
849
+ tgt_sizes = data["tgt_sizes"]
850
+
851
+ device = self.vpm.embeddings.position_embedding.weight.device
852
+ dtype = self.vpm.embeddings.position_embedding.weight.dtype
853
+ all_pixel_values_lst = [
854
+ i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
855
+ ]
856
+
857
+ max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item()
858
+ assert isinstance(max_patches, int)
859
+
860
+ all_pixel_values = torch.nn.utils.rnn.pad_sequence(
861
+ all_pixel_values_lst, batch_first=True, padding_value=0.0)
862
+ B, L, _ = all_pixel_values.shape
863
+ all_pixel_values = all_pixel_values.permute(0, 2,
864
+ 1).reshape(B, 3, -1, L)
865
+
866
+ patch_attn_mask = torch.zeros((B, 1, max_patches),
867
+ dtype=torch.bool,
868
+ device=device)
869
+ for i in range(B):
870
+ patch_attn_mask[i, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True
871
+
872
+ return self.get_vision_embedding(all_pixel_values.type(dtype),
873
+ patch_attn_mask, tgt_sizes)
874
+
875
+ def is_default_weight_loading(self, name: str) -> bool:
876
+ return "resampler" in name
877
+
878
+
879
+# NOTE: Currently, information about this model is unavailable. We are
880
+# temporarily using `MiniCPMVQwen2` as it's name. The name may need
881
+# to be modified in the future.
882
+class MiniCPMVQwen2(MiniCPMVBaseModel):
883
+
884
+ def __init__(
885
+ self,
886
+ config: PretrainedConfig,
887
+ multimodal_config: MultiModalConfig,
888
+ cache_config: Optional[CacheConfig] = None,
889
+ quant_config: Optional[QuantizationConfig] = None,
890
+ ):
891
+ super().__init__(config, multimodal_config, cache_config, quant_config)
892
+
893
+ def init_llm(
894
+ self,
895
+ config: PretrainedConfig,
896
+ cache_config: Optional[CacheConfig] = None,
897
+ quant_config: Optional[QuantizationConfig] = None,
898
+ ) -> nn.Module:
899
+ return Qwen2Model(config,
900
+ cache_config=cache_config,
901
+ quant_config=quant_config)
902
+
903
+ def init_vision_module(self) -> nn.Module:
904
+ # A custom version of SiglipVisionTransformer, won't work with TP
905
+ from vllm.model_executor.models.na_vit import SiglipVisionTransformer
906
+
907
+ if self.config._attn_implementation == "flash_attention_2":
908
+ self.config.vision_config._attn_implementation = "flash_attention_2"
909
+ else:
910
+ # not support sdpa
911
+ self.config.vision_config._attn_implementation = "eager"
912
+ model = SiglipVisionTransformer(self.config.vision_config)
913
+ if self.config.drop_vision_last_layer:
914
+ model.encoder.layers = model.encoder.layers[:-1]
915
+ return model
916
+
917
+ def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
918
+ with set_default_torch_dtype(torch.float16):
919
+ resampler = Resampler2_5(
920
+ num_queries=self.config.query_num,
921
+ embed_dim=embed_dim,
922
+ num_heads=embed_dim // 128,
923
+ kv_dim=vision_dim,
924
+ )
925
+
926
+ return resampler
927
+
928
+ def get_vision_embedding(
929
+ self,
930
+ pixel_values: List[torch.Tensor],
931
+ patch_attn_mask: Optional[torch.Tensor] = None,
932
+ tgt_sizes: Optional[torch.Tensor] = None,
933
+ ) -> torch.Tensor:
934
+ vision_embedding = self.vpm(
935
+ pixel_values,
936
+ patch_attention_mask=patch_attn_mask,
937
+ tgt_sizes=tgt_sizes,
938
+ ).last_hidden_state
939
+ return vision_embedding
940
+
941
+ def get_vision_hidden_states(self,
942
+ data: MiniCPMVImageInputs) -> torch.Tensor:
943
+ pixel_values = data["pixel_values"]
944
+ tgt_sizes = data["tgt_sizes"]
945
+
946
+ device = self.vpm.embeddings.position_embedding.weight.device
947
+ dtype = self.vpm.embeddings.position_embedding.weight.dtype
948
+ all_pixel_values_lst = [
949
+ i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
950
+ ]
951
+
952
+ max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item()
953
+ assert isinstance(max_patches, int)
954
+
955
+ all_pixel_values = torch.nn.utils.rnn.pad_sequence(
956
+ all_pixel_values_lst, batch_first=True, padding_value=0.0)
957
+ B, L, _ = all_pixel_values.shape
958
+ all_pixel_values = all_pixel_values.permute(0, 2,
959
+ 1).reshape(B, 3, -1, L)
960
+
961
+ patch_attn_mask = torch.zeros((B, 1, max_patches),
962
+ dtype=torch.bool,
963
+ device=device)
964
+ for i in range(B):
965
+ patch_attn_mask[i, 0, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True
966
+ vision_embedding = self.vpm(
967
+ all_pixel_values.type(dtype),
968
+ patch_attention_mask=patch_attn_mask,
969
+ tgt_sizes=tgt_sizes,
970
+ ).last_hidden_state
971
+
972
+ return self.resampler(vision_embedding, tgt_sizes)
973
+
974
+ def is_default_weight_loading(self, name: str) -> bool:
975
+ return "resampler" in name or "vpm" in name
976
+
977
+
978
+@MULTIMODAL_REGISTRY.register_image_input_mapper()
979
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
980
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
981
+@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
982
+class MiniCPMV(MiniCPMVBaseModel):
983
+ """
984
+ Different versions of MiniCPMV use different visual encoders and LLMs,
985
+ which is not conducive to the current integration logic of LoRA and
986
+ bitsandbytes in vLLM. Therefore, it is necessary to separate them.
987
+ """
988
+
989
+ def __new__(
990
+ cls,
991
+ config: PretrainedConfig,
992
+ multimodal_config: MultiModalConfig,
993
+ cache_config: Optional[CacheConfig] = None,
994
+ quant_config: Optional[QuantizationConfig] = None,
995
+ ):
996
+ if not hasattr(config, "version"):
997
+ if config.hidden_size == 2304 and config.query_num == 64:
998
+ version = (2, 0)
999
+ else:
1000
+ version = (2, 5)
1001
+ else:
1002
+ version = str(config.version).split(".")
1003
+ version = tuple([int(x) for x in version])
1004
+ # Dispatch class based on version
1005
+ if version == (2, 0):
1006
+ instance_class = MiniCPMV2
1007
+ elif version == (2, 5):
1008
+ instance_class = MiniCPMV2_5
1009
+ else:
1010
+ instance_class = MiniCPMVQwen2
1011
+ return instance_class(config, multimodal_config, cache_config,
1012
+ quant_config)
vllm/model_executor/models/na_vit.py
+1
-1
@@ -100,7 +100,7 @@ def _get_unpad_data(attention_mask):
100
indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
101
max_seqlen_in_batch = seqlens_in_batch.max().item()
102
cu_seqlens = F.pad(
103
- torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
103
+ torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
104
return (
105
indices,
106
cu_seqlens,