A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model][LoRA]LoRA support added for MiniCPMV2.6 (#8943)
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Jee Jee Li committed
Sep 30, 2024 at 12:31 UTC
8e60afa15eb9a0540ce6c453b974a945adff3320
3 files changed
+49
-880
vllm/model_executor/models/idefics2_vision_model.py
+15
-9
@@ -65,11 +65,10 @@ class Idefics2VisionEmbeddings(nn.Module):
65
self.position_embedding = nn.Embedding(self.num_positions,
66
self.embed_dim)
67
68
- def forward(
69
- self,
70
- pixel_values: torch.FloatTensor,
71
- patch_attention_mask: torch.BoolTensor,
72
- ) -> torch.Tensor:
68
+ def forward(self,
69
+ pixel_values: torch.FloatTensor,
70
+ patch_attention_mask: torch.BoolTensor,
71
+ tgt_sizes: Optional[torch.IntTensor] = None) -> torch.Tensor:
72
batch_size, _, max_im_h, max_im_w = pixel_values.shape
73
patch_embeds = self.patch_embedding(pixel_values)
74
embeddings = patch_embeds.flatten(2).transpose(1, 2)
@@ -84,8 +83,13 @@ class Idefics2VisionEmbeddings(nn.Module):
83
fill_value=0)
84
85
for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
87
- nb_patches_h = p_attn_mask[:, 0].sum()
88
- nb_patches_w = p_attn_mask[0].sum()
86
+
87
+ if tgt_sizes is not None:
88
+ nb_patches_h = tgt_sizes[batch_idx][0]
89
+ nb_patches_w = tgt_sizes[batch_idx][1]
90
+ else:
91
+ nb_patches_h = p_attn_mask[:, 0].sum()
92
+ nb_patches_w = p_attn_mask[0].sum()
93
fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
94
fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
95
bucket_coords_h = torch.bucketize(fractional_coords_h,
@@ -287,10 +291,12 @@ class Idefics2VisionTransformer(nn.Module):
291
self,
292
pixel_values,
293
patch_attention_mask: Optional[torch.BoolTensor] = None,
290
- ) -> torch.tensor:
294
+ tgt_sizes: Optional[torch.IntTensor] = None,
295
+ ) -> torch.Tensor:
296
hidden_states = self.embeddings(
297
pixel_values=pixel_values,
293
- patch_attention_mask=patch_attention_mask)
298
+ patch_attention_mask=patch_attention_mask,
299
+ tgt_sizes=tgt_sizes)
300
encoder_outputs = self.encoder(hidden_states)
301
last_hidden_state = self.post_layernorm(encoder_outputs)
302
return last_hidden_state
vllm/model_executor/models/minicpmv.py
+34
-67
@@ -31,17 +31,15 @@ import torch
31
import torch.types
32
from PIL import Image
33
from torch import nn
34
-from torch.nn.init import trunc_normal_
34
from transformers import PretrainedConfig
35
from typing_extensions import NotRequired
36
37
from vllm.attention import AttentionMetadata
38
from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
39
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
41
-from vllm.model_executor.layers.linear import ReplicatedLinear
40
from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
from vllm.model_executor.layers.quantization import QuantizationConfig
44
-from vllm.model_executor.layers.resampler import (Resampler2,
42
+from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
43
get_2d_sincos_pos_embed)
44
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
@@ -106,58 +104,6 @@ class MiniCPMVImagePixelInputs(TypedDict):
104
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
105
106
109
-class BaseResampler(nn.Module):
110
- """
111
- A 2D perceiver-resampler network with one cross attention layers by
112
- (grid_size**2) learnable queries and 2d sincos pos_emb
113
- Outputs:
114
- A tensor with the shape of (grid_size**2, embed_dim)
115
- """
116
-
117
- def __init__(
118
- self,
119
- num_queries: int,
120
- embed_dim: int,
121
- num_heads: int,
122
- kv_dim: Optional[int] = None,
123
- norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
124
- ) -> None:
125
- super().__init__()
126
-
127
- self.num_queries = num_queries
128
- self.embed_dim = embed_dim
129
- self.num_heads = num_heads
130
-
131
- self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
132
- trunc_normal_(self.query, std=0.02)
133
- if kv_dim is not None and kv_dim != embed_dim:
134
- self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
135
- else:
136
- # Maintain the same return value with ReplicatedLinear.forward
137
- self.kv_proj = lambda *args, **kwargs: (
138
- nn.Identity()(*args, **kwargs),
139
- None,
140
- )
141
- self.attn = nn.MultiheadAttention(embed_dim, num_heads)
142
- self.ln_q = norm_layer(embed_dim)
143
- self.ln_kv = norm_layer(embed_dim)
144
- self.ln_post = norm_layer(embed_dim)
145
- self.proj = nn.Parameter(
146
- (embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
147
-
148
- def _init_weights(self, m: nn.Module) -> None:
149
- if isinstance(m, nn.Linear):
150
- trunc_normal_(m.weight, std=0.02)
151
- if isinstance(m, nn.Linear) and m.bias is not None:
152
- nn.init.constant_(m.bias, 0)
153
- elif isinstance(m, nn.LayerNorm):
154
- nn.init.constant_(m.bias, 0)
155
- nn.init.constant_(m.weight, 1.0)
156
-
157
- def _repeat(self, query, N: int):
158
- return query.unsqueeze(1).repeat(1, N, 1)
159
-
160
-
107
class Resampler2_5(BaseResampler):
108
109
def __init__(
@@ -869,7 +815,35 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
815
return "resampler" in name
816
817
872
-class MiniCPMV2_6(MiniCPMVBaseModel):
818
+class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
819
+ packed_modules_mapping = {
820
+ "qkv_proj": [
821
+ "q_proj",
822
+ "k_proj",
823
+ "v_proj",
824
+ ],
825
+ "gate_up_proj": [
826
+ "gate_proj",
827
+ "up_proj",
828
+ ],
829
+ }
830
+ # LoRA specific attributes
831
+ supported_lora_modules = [
832
+ # vision encoder
833
+ "fc1",
834
+ "fc2",
835
+ "out_proj",
836
+ # language model
837
+ "qkv_proj", # same name with vision encoder
838
+ "o_proj",
839
+ "gate_up_proj",
840
+ "down_proj",
841
+ # resampler
842
+ "kv_proj",
843
+ ]
844
+
845
+ embedding_modules = {}
846
+ embedding_padding_modules = []
847
848
def __init__(
849
self,
@@ -894,15 +868,8 @@ class MiniCPMV2_6(MiniCPMVBaseModel):
868
name="model")
869
870
def init_vision_module(self) -> nn.Module:
897
- # A custom version of SiglipVisionTransformer, won't work with TP
898
- from vllm.model_executor.models.na_vit import SiglipVisionTransformer
871
900
- if self.config._attn_implementation == "flash_attention_2":
901
- self.config.vision_config._attn_implementation = "flash_attention_2"
902
- else:
903
- # not support sdpa
904
- self.config.vision_config._attn_implementation = "eager"
905
- model = SiglipVisionTransformer(self.config.vision_config)
872
+ model = Idefics2VisionTransformer(self.config.vision_config)
873
if self.config.drop_vision_last_layer:
874
model.encoder.layers = model.encoder.layers[:-1]
875
return model
@@ -928,7 +895,7 @@ class MiniCPMV2_6(MiniCPMVBaseModel):
895
pixel_values,
896
patch_attention_mask=patch_attn_mask,
897
tgt_sizes=tgt_sizes,
931
- ).last_hidden_state
898
+ )
899
return vision_embedding
900
901
def get_vision_hidden_states(
@@ -960,12 +927,12 @@ class MiniCPMV2_6(MiniCPMVBaseModel):
927
all_pixel_values.type(dtype),
928
patch_attention_mask=patch_attn_mask,
929
tgt_sizes=tgt_sizes,
963
- ).last_hidden_state
930
+ )
931
932
return self.resampler(vision_embedding, tgt_sizes)
933
934
def is_default_weight_loading(self, name: str) -> bool:
968
- return "resampler" in name or "vpm" in name
935
+ return "resampler" in name
936
937
938
_SUPPORT_VERSION = {
vllm/model_executor/models/na_vit.py
deleted
-804
@@ -1,804 +0,0 @@
1
-import logging
2
-import math
3
-import os
4
-import warnings
5
-from typing import Optional, Tuple, Union
6
-
7
-import numpy as np
8
-import torch
9
-import torch.nn.functional as F
10
-from torch import nn
11
-from torch.nn.init import _calculate_fan_in_and_fan_out
12
-from transformers.activations import ACT2FN
13
-from transformers.configuration_utils import PretrainedConfig
14
-from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
15
-from transformers.modeling_outputs import (BaseModelOutput,
16
- BaseModelOutputWithPooling)
17
-from transformers.modeling_utils import PreTrainedModel
18
-from transformers.utils import (ModelOutput, is_flash_attn_2_available,
19
- replace_return_docstrings)
20
-
21
-logger = logging.getLogger("vllm")
22
-
23
-
24
-# For Siglip: copied from
25
-# HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit and add tgt_sizes
26
-# Remove hints as there's little possibility to change these code.
27
-class SiglipVisionConfig(PretrainedConfig):
28
-
29
- model_type = "siglip_vision_model"
30
-
31
- def __init__(
32
- self,
33
- hidden_size=768,
34
- intermediate_size=3072,
35
- num_hidden_layers=12,
36
- num_attention_heads=12,
37
- num_channels=3,
38
- image_size=224,
39
- patch_size=16,
40
- hidden_act="gelu_pytorch_tanh",
41
- layer_norm_eps=1e-6,
42
- attention_dropout=0.0,
43
- **kwargs,
44
- ):
45
- super().__init__(**kwargs)
46
-
47
- self.hidden_size = hidden_size
48
- self.intermediate_size = intermediate_size
49
- self.num_hidden_layers = num_hidden_layers
50
- self.num_attention_heads = num_attention_heads
51
- self.num_channels = num_channels
52
- self.patch_size = patch_size
53
- self.image_size = image_size
54
- self.attention_dropout = attention_dropout
55
- self.layer_norm_eps = layer_norm_eps
56
- self.hidden_act = hidden_act
57
-
58
- @classmethod
59
- def from_pretrained(cls, pretrained_model_name_or_path: Union[str,
60
- os.PathLike],
61
- **kwargs) -> "PretrainedConfig":
62
- cls._set_token_in_kwargs(kwargs)
63
-
64
- config_dict, kwargs = cls.get_config_dict(
65
- pretrained_model_name_or_path, **kwargs)
66
-
67
- # get the vision config dict if we are loading from SiglipConfig
68
- if config_dict.get("model_type") == "siglip":
69
- config_dict = config_dict["vision_config"]
70
-
71
- if "model_type" in config_dict and hasattr(
72
- cls,
73
- "model_type") and config_dict["model_type"] != cls.model_type:
74
- logger.warning(
75
- "You are using a model of type %s to "
76
- "instantiate a model of type %s. "
77
- "This is not supported for all configurations"
78
- "of models and can yield errors.", config_dict['model_type'],
79
- cls.model_type)
80
-
81
- return cls.from_dict(config_dict, **kwargs)
82
-
83
-
84
-_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
85
-
86
-SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
87
- "google/siglip-base-patch16-224",
88
- # See all SigLIP models at https://huggingface.co/models?filter=siglip
89
-]
90
-
91
-if is_flash_attn_2_available():
92
- from flash_attn import flash_attn_func, flash_attn_varlen_func
93
- from flash_attn.bert_padding import pad_input # noqa
94
- from flash_attn.bert_padding import index_first_axis, unpad_input
95
-
96
-
97
-# Copied from transformers.models.llama.modeling_llama._get_unpad_data
98
-def _get_unpad_data(attention_mask):
99
- seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
100
- indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
101
- max_seqlen_in_batch = seqlens_in_batch.max().item()
102
- cu_seqlens = F.pad(
103
- torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
104
- return (
105
- indices,
106
- cu_seqlens,
107
- max_seqlen_in_batch,
108
- )
109
-
110
-
111
-def _trunc_normal_(tensor, mean, std, a, b):
112
-
113
- def norm_cdf(x):
114
- # Computes standard normal cumulative distribution function
115
- return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
116
-
117
- if (mean < a - 2 * std) or (mean > b + 2 * std):
118
- warnings.warn(
119
- "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
120
- "The distribution of values may be incorrect.",
121
- stacklevel=2,
122
- )
123
-
124
- # Values are generated by using a truncated uniform distribution and
125
- # then using the inverse CDF for the normal distribution.
126
- # Get upper and lower cdf values
127
- l_ = norm_cdf((a - mean) / std)
128
- u = norm_cdf((b - mean) / std)
129
-
130
- # Uniformly fill tensor with values from [l, u], then translate to
131
- # [2l-1, 2u-1].
132
- tensor.uniform_(2 * l_ - 1, 2 * u - 1)
133
-
134
- # Use inverse cdf transform for normal distribution to get truncated
135
- # standard normal
136
- if tensor.dtype in [torch.float16, torch.bfloat16]:
137
- # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
138
- og_dtype = tensor.dtype
139
- tensor = tensor.to(torch.float32)
140
- tensor.erfinv_()
141
- tensor = tensor.to(og_dtype)
142
- else:
143
- tensor.erfinv_()
144
-
145
- # Transform to proper mean, std
146
- tensor.mul_(std * math.sqrt(2.0))
147
- tensor.add_(mean)
148
-
149
- # Clamp to ensure it's in the proper range
150
- if tensor.dtype == torch.float16:
151
- # The `clamp_` op is not (yet?) defined in float16+cpu
152
- tensor = tensor.to(torch.float32)
153
- tensor.clamp_(min=a, max=b)
154
- tensor = tensor.to(torch.float16)
155
- else:
156
- tensor.clamp_(min=a, max=b)
157
-
158
-
159
-def trunc_normal_tf_(tensor: torch.Tensor,
160
- mean: float = 0.0,
161
- std: float = 1.0,
162
- a: float = -2.0,
163
- b: float = 2.0) -> torch.Tensor:
164
- with torch.no_grad():
165
- _trunc_normal_(tensor, 0, 1.0, a, b)
166
- tensor.mul_(std).add_(mean)
167
-
168
-
169
-def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
170
- fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
171
- if mode == "fan_in":
172
- denom = fan_in
173
- elif mode == "fan_out":
174
- denom = fan_out
175
- elif mode == "fan_avg":
176
- denom = (fan_in + fan_out) / 2
177
-
178
- variance = scale / denom
179
-
180
- if distribution == "truncated_normal":
181
- # constant is stddev of standard normal truncated to (-2, 2)
182
- trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
183
- elif distribution == "normal":
184
- with torch.no_grad():
185
- tensor.normal_(std=math.sqrt(variance))
186
- elif distribution == "uniform":
187
- bound = math.sqrt(3 * variance)
188
- with torch.no_grad():
189
- tensor.uniform_(-bound, bound)
190
- else:
191
- raise ValueError(f"invalid distribution {distribution}")
192
-
193
-
194
-def lecun_normal_(tensor):
195
- variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
196
-
197
-
198
-def default_flax_embed_init(tensor):
199
- variance_scaling_(tensor, mode="fan_in", distribution="normal")
200
-
201
-
202
-class SiglipVisionModelOutput(ModelOutput):
203
- image_embeds: Optional[torch.FloatTensor] = None
204
- last_hidden_state: torch.FloatTensor = None
205
- hidden_states: Optional[Tuple[torch.FloatTensor]] = None
206
- attentions: Optional[Tuple[torch.FloatTensor]] = None
207
-
208
-
209
-class SiglipVisionEmbeddings(nn.Module):
210
-
211
- def __init__(self, config: SiglipVisionConfig):
212
- super().__init__()
213
- self.config = config
214
- self.embed_dim = config.hidden_size
215
- self.image_size = config.image_size
216
- self.patch_size = config.patch_size
217
-
218
- self.patch_embedding = nn.Conv2d(
219
- in_channels=config.num_channels,
220
- out_channels=self.embed_dim,
221
- kernel_size=self.patch_size,
222
- stride=self.patch_size,
223
- padding="valid",
224
- )
225
-
226
- self.num_patches_per_side = self.image_size // self.patch_size
227
- self.num_patches = self.num_patches_per_side**2
228
- self.num_positions = self.num_patches
229
- self.position_embedding = nn.Embedding(self.num_positions,
230
- self.embed_dim)
231
-
232
- def forward(self,
233
- pixel_values: torch.FloatTensor,
234
- patch_attention_mask: torch.BoolTensor,
235
- tgt_sizes: Optional[torch.IntTensor] = None) -> torch.Tensor:
236
- batch_size = pixel_values.size(0)
237
-
238
- patch_embeds = self.patch_embedding(pixel_values)
239
- embeddings = patch_embeds.flatten(2).transpose(1, 2)
240
-
241
- max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3)
242
- max_nb_patches_h, max_nb_patches_w = (max_im_h // self.patch_size,
243
- max_im_w // self.patch_size)
244
- boundaries = torch.arange(1 / self.num_patches_per_side, 1.0,
245
- 1 / self.num_patches_per_side)
246
- position_ids = torch.full(
247
- size=(
248
- batch_size,
249
- max_nb_patches_h * max_nb_patches_w,
250
- ),
251
- fill_value=0,
252
- )
253
-
254
- for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
255
- if tgt_sizes is not None:
256
- nb_patches_h = tgt_sizes[batch_idx][0]
257
- nb_patches_w = tgt_sizes[batch_idx][1]
258
- else:
259
- nb_patches_h = p_attn_mask[:, 0].sum()
260
- nb_patches_w = p_attn_mask[0].sum()
261
-
262
- fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
263
- fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
264
-
265
- bucket_coords_h = torch.bucketize(fractional_coords_h,
266
- boundaries,
267
- right=True)
268
- bucket_coords_w = torch.bucketize(fractional_coords_w,
269
- boundaries,
270
- right=True)
271
-
272
- pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side +
273
- bucket_coords_w).flatten()
274
- position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
275
-
276
- position_ids = position_ids.to(self.position_embedding.weight.device)
277
-
278
- embeddings = embeddings + self.position_embedding(position_ids)
279
- return embeddings
280
-
281
-
282
-class SiglipAttention(nn.Module):
283
- """Multi-headed attention from 'Attention Is All You Need' paper"""
284
-
285
- # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
286
- def __init__(self, config):
287
- super().__init__()
288
- self.config = config
289
- self.embed_dim = config.hidden_size
290
- self.num_heads = config.num_attention_heads
291
- self.head_dim = self.embed_dim // self.num_heads
292
- if self.head_dim * self.num_heads != self.embed_dim:
293
- raise ValueError(
294
- "embed_dim must be divisible by num_heads (got `embed_dim`: "
295
- f"{self.embed_dim} and `num_heads`:"
296
- f" {self.num_heads}).")
297
- self.scale = self.head_dim**-0.5
298
- self.dropout = config.attention_dropout
299
-
300
- self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
301
- self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
302
- self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
303
- self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
304
-
305
- def forward(
306
- self,
307
- hidden_states: torch.Tensor,
308
- attention_mask: Optional[torch.Tensor] = None,
309
- output_attentions: Optional[bool] = False,
310
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
311
- Optional[Tuple[torch.Tensor]]]:
312
- """Input shape: Batch x Time x Channel"""
313
-
314
- batch_size, q_len, _ = hidden_states.size()
315
-
316
- query_states = self.q_proj(hidden_states)
317
- key_states = self.k_proj(hidden_states)
318
- value_states = self.v_proj(hidden_states)
319
-
320
- query_states = query_states.view(batch_size, q_len, self.num_heads,
321
- self.head_dim).transpose(1, 2)
322
- key_states = key_states.view(batch_size, q_len, self.num_heads,
323
- self.head_dim).transpose(1, 2)
324
- value_states = value_states.view(batch_size, q_len, self.num_heads,
325
- self.head_dim).transpose(1, 2)
326
-
327
- k_v_seq_len = key_states.shape[-2]
328
- attn_weights = torch.matmul(query_states, key_states.transpose(
329
- 2, 3)) * self.scale
330
-
331
- if attn_weights.size() != (batch_size, self.num_heads, q_len,
332
- k_v_seq_len):
333
- raise ValueError(
334
- "Attention weights should be of size "
335
- f"{(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
336
- f" {attn_weights.size()}")
337
-
338
- if attention_mask is not None:
339
- if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
340
- raise ValueError(
341
- "Attention mask should be of size "
342
- f"{(batch_size, 1, q_len, k_v_seq_len)}",
343
- f"but is {attention_mask.size()}")
344
- attn_weights = attn_weights + attention_mask
345
-
346
- # upcast attention to fp32
347
- attn_weights = nn.functional.softmax(attn_weights,
348
- dim=-1,
349
- dtype=torch.float32).to(
350
- query_states.dtype)
351
- attn_weights = nn.functional.dropout(attn_weights,
352
- p=self.dropout,
353
- training=self.training)
354
- attn_output = torch.matmul(attn_weights, value_states)
355
-
356
- if attn_output.size() != (batch_size, self.num_heads, q_len,
357
- self.head_dim):
358
- raise ValueError(
359
- "`attn_output` should be of size "
360
- f"{(batch_size, self.num_heads, q_len, self.head_dim)}, "
361
- "but is"
362
- f" {attn_output.size()}")
363
-
364
- attn_output = attn_output.transpose(1, 2).contiguous()
365
- attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
366
-
367
- attn_output = self.out_proj(attn_output)
368
-
369
- return attn_output, attn_weights
370
-
371
-
372
-class SiglipFlashAttention2(SiglipAttention):
373
-
374
- def __init__(self, *args, **kwargs):
375
- super().__init__(*args, **kwargs)
376
- self.is_causal = False # Hack to make sure we don't use a causal mask
377
-
378
- def forward(
379
- self,
380
- hidden_states: torch.Tensor,
381
- attention_mask: Optional[torch.LongTensor] = None,
382
- position_ids: Optional[torch.LongTensor] = None,
383
- past_key_value: Optional[Tuple[torch.Tensor]] = None,
384
- output_attentions: bool = False,
385
- use_cache: bool = False,
386
- **kwargs,
387
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
388
- Optional[Tuple[torch.Tensor]]]:
389
- output_attentions = False
390
-
391
- bsz, q_len, _ = hidden_states.size()
392
-
393
- query_states = self.q_proj(hidden_states)
394
- key_states = self.k_proj(hidden_states)
395
- value_states = self.v_proj(hidden_states)
396
-
397
- query_states = query_states.view(bsz, q_len, self.num_heads,
398
- self.head_dim).transpose(1, 2)
399
- key_states = key_states.view(bsz, q_len, self.num_heads,
400
- self.head_dim).transpose(1, 2)
401
- value_states = value_states.view(bsz, q_len, self.num_heads,
402
- self.head_dim).transpose(1, 2)
403
-
404
- kv_seq_len = key_states.shape[-2]
405
- if past_key_value is not None:
406
- kv_seq_len += past_key_value.get_usable_length(
407
- kv_seq_len, self.layer_idx)
408
-
409
- query_states = query_states.transpose(1, 2)
410
- key_states = key_states.transpose(1, 2)
411
- value_states = value_states.transpose(1, 2)
412
-
413
- dropout_rate = self.dropout if self.training else 0.0
414
-
415
- input_dtype = query_states.dtype
416
- if input_dtype == torch.float32:
417
- if torch.is_autocast_enabled():
418
- target_dtype = torch.get_autocast_gpu_dtype()
419
- # Handle the case where the model is quantized
420
- elif hasattr(self.config, "_pre_quantization_dtype"):
421
- target_dtype = self.config._pre_quantization_dtype
422
- else:
423
- target_dtype = self.q_proj.weight.dtype
424
-
425
- logger.warning(
426
- "The input hidden states seems to be "
427
- "silently casted in float32, "
428
- "this might be related to the fact "
429
- "you have upcasted embedding or layer norm layers in float32. "
430
- "We will cast back the input in"
431
- " %s.", target_dtype)
432
-
433
- query_states = query_states.to(target_dtype)
434
- key_states = key_states.to(target_dtype)
435
- value_states = value_states.to(target_dtype)
436
-
437
- attn_output = self._flash_attention_forward(query_states,
438
- key_states,
439
- value_states,
440
- attention_mask,
441
- q_len,
442
- dropout=dropout_rate)
443
-
444
- attn_output = attn_output.reshape(bsz, q_len,
445
- self.embed_dim).contiguous()
446
- attn_output = self.out_proj(attn_output)
447
-
448
- if not output_attentions:
449
- attn_weights = None
450
-
451
- return attn_output, attn_weights
452
-
453
- def _flash_attention_forward(self,
454
- query_states,
455
- key_states,
456
- value_states,
457
- attention_mask,
458
- query_length,
459
- dropout=0.0,
460
- softmax_scale=None):
461
- causal = self.is_causal and query_length != 1
462
-
463
- # Contains at least one padding token in the sequence
464
- if attention_mask is not None:
465
- batch_size = query_states.shape[0]
466
- (query_states, key_states, value_states, indices_q, cu_seq_lens,
467
- max_seq_lens) = self._upad_input(query_states, key_states,
468
- value_states, attention_mask,
469
- query_length)
470
-
471
- cu_seqlens_q, cu_seqlens_k = cu_seq_lens
472
- max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
473
-
474
- attn_output_unpad = flash_attn_varlen_func(
475
- query_states,
476
- key_states,
477
- value_states,
478
- cu_seqlens_q=cu_seqlens_q,
479
- cu_seqlens_k=cu_seqlens_k,
480
- max_seqlen_q=max_seqlen_in_batch_q,
481
- max_seqlen_k=max_seqlen_in_batch_k,
482
- dropout_p=dropout,
483
- softmax_scale=softmax_scale,
484
- causal=causal,
485
- )
486
-
487
- attn_output = pad_input(attn_output_unpad, indices_q, batch_size,
488
- query_length)
489
- else:
490
- attn_output = flash_attn_func(query_states,
491
- key_states,
492
- value_states,
493
- dropout,
494
- softmax_scale=softmax_scale,
495
- causal=causal)
496
-
497
- return attn_output
498
-
499
- def _upad_input(self, query_layer, key_layer, value_layer, attention_mask,
500
- query_length):
501
- indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(
502
- attention_mask)
503
- batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
504
-
505
- key_layer = index_first_axis(
506
- key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
507
- head_dim), indices_k)
508
- value_layer = index_first_axis(
509
- value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
510
- head_dim), indices_k)
511
- if query_length == kv_seq_len:
512
- query_layer = index_first_axis(
513
- query_layer.reshape(batch_size * kv_seq_len, self.num_heads,
514
- head_dim), indices_k)
515
- cu_seqlens_q = cu_seqlens_k
516
- max_seqlen_in_batch_q = max_seqlen_in_batch_k
517
- indices_q = indices_k
518
- elif query_length == 1:
519
- max_seqlen_in_batch_q = 1
520
- cu_seqlens_q = torch.arange(
521
- batch_size + 1, dtype=torch.int32, device=query_layer.device
522
- ) # There is a memcpy here, that is very bad.
523
- indices_q = cu_seqlens_q[:-1]
524
- query_layer = query_layer.squeeze(1)
525
- else:
526
- # The -q_len: slice assumes left padding.
527
- attention_mask = attention_mask[:, -query_length:]
528
- (query_layer, indices_q, cu_seqlens_q,
529
- max_seqlen_in_batch_q) = unpad_input(query_layer, attention_mask)
530
-
531
- return (
532
- query_layer,
533
- key_layer,
534
- value_layer,
535
- indices_q,
536
- (cu_seqlens_q, cu_seqlens_k),
537
- (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
538
- )
539
-
540
-
541
-# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
542
-class SiglipMLP(nn.Module):
543
-
544
- def __init__(self, config):
545
- super().__init__()
546
- self.config = config
547
- self.activation_fn = ACT2FN[config.hidden_act]
548
- self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
549
- self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
550
-
551
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
552
- hidden_states = self.fc1(hidden_states)
553
- hidden_states = self.activation_fn(hidden_states)
554
- hidden_states = self.fc2(hidden_states)
555
- return hidden_states
556
-
557
-
558
-# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer
559
-# with CLIP->Siglip
560
-class SiglipEncoderLayer(nn.Module):
561
-
562
- def __init__(self, config: SiglipVisionConfig):
563
- super().__init__()
564
- self.embed_dim = config.hidden_size
565
- self._use_flash_attention_2 = (
566
- config._attn_implementation == "flash_attention_2")
567
- self.self_attn = (SiglipAttention(config)
568
- if not self._use_flash_attention_2 else
569
- SiglipFlashAttention2(config))
570
- self.layer_norm1 = nn.LayerNorm(self.embed_dim,
571
- eps=config.layer_norm_eps)
572
- self.mlp = SiglipMLP(config)
573
- self.layer_norm2 = nn.LayerNorm(self.embed_dim,
574
- eps=config.layer_norm_eps)
575
-
576
- def forward(
577
- self,
578
- hidden_states: torch.Tensor,
579
- attention_mask: torch.Tensor,
580
- output_attentions: Optional[bool] = False,
581
- ) -> Tuple[torch.FloatTensor]:
582
- residual = hidden_states
583
-
584
- hidden_states = self.layer_norm1(hidden_states)
585
- hidden_states, attn_weights = self.self_attn(
586
- hidden_states=hidden_states,
587
- attention_mask=attention_mask,
588
- output_attentions=output_attentions,
589
- )
590
- hidden_states = residual + hidden_states
591
-
592
- residual = hidden_states
593
- hidden_states = self.layer_norm2(hidden_states)
594
- hidden_states = self.mlp(hidden_states)
595
- hidden_states = residual + hidden_states
596
-
597
- outputs = (hidden_states, )
598
-
599
- if output_attentions:
600
- outputs += (attn_weights, )
601
-
602
- return outputs
603
-
604
-
605
-class SiglipPreTrainedModel(PreTrainedModel):
606
- config_class = SiglipVisionConfig
607
- base_model_prefix = "siglip"
608
- supports_gradient_checkpointing = True
609
-
610
- def _init_weights(self, module):
611
- """Initialize the weights"""
612
-
613
- if isinstance(module, SiglipVisionEmbeddings):
614
- width = self.config.hidden_size
615
- nn.init.normal_(module.position_embedding.weight,
616
- std=1 / np.sqrt(width))
617
- elif isinstance(module, nn.Embedding):
618
- default_flax_embed_init(module.weight)
619
- elif isinstance(module, SiglipAttention):
620
- nn.init.normal_(module.q_proj.weight)
621
- nn.init.normal_(module.k_proj.weight)
622
- nn.init.normal_(module.v_proj.weight)
623
- nn.init.normal_(module.out_proj.weight)
624
- nn.init.zeros_(module.q_proj.bias)
625
- nn.init.zeros_(module.k_proj.bias)
626
- nn.init.zeros_(module.v_proj.bias)
627
- nn.init.zeros_(module.out_proj.bias)
628
- elif isinstance(module, SiglipMLP):
629
- nn.init.normal_(module.fc1.weight)
630
- nn.init.normal_(module.fc2.weight)
631
- nn.init.normal_(module.fc1.bias, std=1e-6)
632
- nn.init.normal_(module.fc2.bias, std=1e-6)
633
- elif isinstance(module, (nn.Linear, nn.Conv2d)):
634
- lecun_normal_(module.weight)
635
- if module.bias is not None:
636
- nn.init.zeros_(module.bias)
637
- elif isinstance(module, nn.LayerNorm):
638
- module.bias.data.zero_()
639
- module.weight.data.fill_(1.0)
640
-
641
-
642
-# Copied from transformers.models.clip.modeling_clip.CLIPEncoder
643
-# with CLIP->Siglip
644
-class SiglipEncoder(nn.Module):
645
-
646
- def __init__(self, config: SiglipVisionConfig):
647
- super().__init__()
648
- self.config = config
649
- self.layers = nn.ModuleList([
650
- SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)
651
- ])
652
- self.gradient_checkpointing = False
653
-
654
- # Ignore copy
655
- def forward(
656
- self,
657
- inputs_embeds,
658
- attention_mask: Optional[torch.Tensor] = None,
659
- output_attentions: Optional[bool] = None,
660
- output_hidden_states: Optional[bool] = None,
661
- return_dict: Optional[bool] = None,
662
- ) -> Union[Tuple, BaseModelOutput]:
663
- output_attentions = output_attentions if output_attentions is not None \
664
- else self.config.output_attentions
665
- output_hidden_states = (output_hidden_states
666
- if output_hidden_states is not None else
667
- self.config.output_hidden_states)
668
- return_dict = return_dict if return_dict is not None \
669
- else self.config.use_return_dict
670
-
671
- encoder_states = () if output_hidden_states else None
672
- all_attentions = () if output_attentions else None
673
-
674
- hidden_states = inputs_embeds
675
- for encoder_layer in self.layers:
676
- if output_hidden_states:
677
- encoder_states = encoder_states + (hidden_states, )
678
- if self.gradient_checkpointing and self.training:
679
- layer_outputs = self._gradient_checkpointing_func(
680
- encoder_layer.__call__,
681
- hidden_states,
682
- attention_mask,
683
- output_attentions,
684
- )
685
- else:
686
- layer_outputs = encoder_layer(
687
- hidden_states,
688
- attention_mask,
689
- output_attentions=output_attentions,
690
- )
691
-
692
- hidden_states = layer_outputs[0]
693
-
694
- if output_attentions:
695
- all_attentions = all_attentions + (layer_outputs[1], )
696
-
697
- if output_hidden_states:
698
- encoder_states = encoder_states + (hidden_states, )
699
-
700
- if not return_dict:
701
- return tuple(
702
- v for v in [hidden_states, encoder_states, all_attentions]
703
- if v is not None)
704
- return BaseModelOutput(last_hidden_state=hidden_states,
705
- hidden_states=encoder_states,
706
- attentions=all_attentions)
707
-
708
-
709
-class SiglipVisionTransformer(SiglipPreTrainedModel):
710
- config_class = SiglipVisionConfig
711
- main_input_name = "pixel_values"
712
- _supports_flash_attn_2 = True
713
-
714
- def __init__(self, config: SiglipVisionConfig):
715
- super().__init__(config)
716
- self.config = config
717
- embed_dim = config.hidden_size
718
-
719
- self.embeddings = SiglipVisionEmbeddings(config)
720
- self.encoder = SiglipEncoder(config)
721
- self.post_layernorm = nn.LayerNorm(embed_dim,
722
- eps=config.layer_norm_eps)
723
- self._use_flash_attention_2 = (
724
- config._attn_implementation == "flash_attention_2")
725
-
726
- # Initialize weights and apply final processing
727
- self.post_init()
728
-
729
- def get_input_embeddings(self) -> nn.Module:
730
- return self.embeddings.patch_embedding
731
-
732
- @replace_return_docstrings(output_type=BaseModelOutputWithPooling,
733
- config_class=SiglipVisionConfig)
734
- def forward(
735
- self,
736
- pixel_values,
737
- patch_attention_mask: Optional[torch.BoolTensor] = None,
738
- tgt_sizes: Optional[torch.IntTensor] = None,
739
- output_attentions: Optional[bool] = None,
740
- output_hidden_states: Optional[bool] = None,
741
- return_dict: Optional[bool] = None,
742
- ) -> Union[Tuple, BaseModelOutputWithPooling]:
743
- r"""
744
- Returns:
745
- """
746
- output_attentions = output_attentions if output_attentions is not None \
747
- else self.config.output_attentions
748
- output_hidden_states = (output_hidden_states
749
- if output_hidden_states is not None else
750
- self.config.output_hidden_states)
751
- return_dict = return_dict if return_dict is not None \
752
- else self.config.use_return_dict
753
-
754
- batch_size = pixel_values.size(0)
755
- if patch_attention_mask is None:
756
- patch_attention_mask = torch.ones(
757
- size=(
758
- batch_size,
759
- pixel_values.size(2) // self.config.patch_size,
760
- pixel_values.size(3) // self.config.patch_size,
761
- ),
762
- dtype=torch.bool,
763
- device=pixel_values.device,
764
- )
765
-
766
- hidden_states = self.embeddings(
767
- pixel_values=pixel_values,
768
- patch_attention_mask=patch_attention_mask,
769
- tgt_sizes=tgt_sizes)
770
-
771
- patch_attention_mask = patch_attention_mask.view(batch_size, -1)
772
- # The call to `_upad_input` in `_flash_attention_forward` is expensive
773
- # So when the `patch_attention_mask` is full of 1s
774
- # (i.e. attending to the whole sequence),
775
- # avoiding passing the attention_mask,
776
- # which is equivalent to attending to the full sequence
777
- if not torch.any(~patch_attention_mask):
778
- attention_mask = None
779
- else:
780
- attention_mask = (_prepare_4d_attention_mask(
781
- patch_attention_mask, hidden_states.dtype)
782
- if not self._use_flash_attention_2 else
783
- patch_attention_mask)
784
-
785
- encoder_outputs = self.encoder(
786
- inputs_embeds=hidden_states,
787
- attention_mask=attention_mask,
788
- output_attentions=output_attentions,
789
- output_hidden_states=output_hidden_states,
790
- return_dict=return_dict,
791
- )
792
-
793
- last_hidden_state = encoder_outputs[0]
794
- last_hidden_state = self.post_layernorm(last_hidden_state)
795
-
796
- if not return_dict:
797
- return (last_hidden_state, None) + encoder_outputs[1:]
798
-
799
- return BaseModelOutputWithPooling(
800
- last_hidden_state=last_hidden_state,
801
- pooler_output=None,
802
- hidden_states=encoder_outputs.hidden_states,
803
- attentions=encoder_outputs.attentions,
804
- )