@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8e60afa15e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model][LoRA]LoRA support added for MiniCPMV2.6 (#8943)

Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Jee Jee Li committed Sep 30, 2024 at 12:31 UTC 8e60afa15eb9a0540ce6c453b974a945adff3320
3 files changed +49 -880
vllm/model_executor/models/idefics2_vision_model.py
+15 -9
@@ -65,11 +65,10 @@ class Idefics2VisionEmbeddings(nn.Module):
65 self.position_embedding = nn.Embedding(self.num_positions,
66 self.embed_dim)
67
68 - def forward(
69 - self,
70 - pixel_values: torch.FloatTensor,
71 - patch_attention_mask: torch.BoolTensor,
72 - ) -> torch.Tensor:
68 + def forward(self,
69 + pixel_values: torch.FloatTensor,
70 + patch_attention_mask: torch.BoolTensor,
71 + tgt_sizes: Optional[torch.IntTensor] = None) -> torch.Tensor:
72 batch_size, _, max_im_h, max_im_w = pixel_values.shape
73 patch_embeds = self.patch_embedding(pixel_values)
74 embeddings = patch_embeds.flatten(2).transpose(1, 2)
@@ -84,8 +83,13 @@ class Idefics2VisionEmbeddings(nn.Module):
83 fill_value=0)
84
85 for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
87 - nb_patches_h = p_attn_mask[:, 0].sum()
88 - nb_patches_w = p_attn_mask[0].sum()
86 +
87 + if tgt_sizes is not None:
88 + nb_patches_h = tgt_sizes[batch_idx][0]
89 + nb_patches_w = tgt_sizes[batch_idx][1]
90 + else:
91 + nb_patches_h = p_attn_mask[:, 0].sum()
92 + nb_patches_w = p_attn_mask[0].sum()
93 fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
94 fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
95 bucket_coords_h = torch.bucketize(fractional_coords_h,
@@ -287,10 +291,12 @@ class Idefics2VisionTransformer(nn.Module):
291 self,
292 pixel_values,
293 patch_attention_mask: Optional[torch.BoolTensor] = None,
290 - ) -> torch.tensor:
294 + tgt_sizes: Optional[torch.IntTensor] = None,
295 + ) -> torch.Tensor:
296 hidden_states = self.embeddings(
297 pixel_values=pixel_values,
293 - patch_attention_mask=patch_attention_mask)
298 + patch_attention_mask=patch_attention_mask,
299 + tgt_sizes=tgt_sizes)
300 encoder_outputs = self.encoder(hidden_states)
301 last_hidden_state = self.post_layernorm(encoder_outputs)
302 return last_hidden_state
vllm/model_executor/models/minicpmv.py
+34 -67
@@ -31,17 +31,15 @@ import torch
31 import torch.types
32 from PIL import Image
33 from torch import nn
34 -from torch.nn.init import trunc_normal_
34 from transformers import PretrainedConfig
35 from typing_extensions import NotRequired
36
37 from vllm.attention import AttentionMetadata
38 from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
39 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
41 -from vllm.model_executor.layers.linear import ReplicatedLinear
40 from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 from vllm.model_executor.layers.quantization import QuantizationConfig
44 -from vllm.model_executor.layers.resampler import (Resampler2,
42 +from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
43 get_2d_sincos_pos_embed)
44 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
@@ -106,58 +104,6 @@ class MiniCPMVImagePixelInputs(TypedDict):
104 DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
105
106
109 -class BaseResampler(nn.Module):
110 - """
111 - A 2D perceiver-resampler network with one cross attention layers by
112 - (grid_size**2) learnable queries and 2d sincos pos_emb
113 - Outputs:
114 - A tensor with the shape of (grid_size**2, embed_dim)
115 - """
116 -
117 - def __init__(
118 - self,
119 - num_queries: int,
120 - embed_dim: int,
121 - num_heads: int,
122 - kv_dim: Optional[int] = None,
123 - norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
124 - ) -> None:
125 - super().__init__()
126 -
127 - self.num_queries = num_queries
128 - self.embed_dim = embed_dim
129 - self.num_heads = num_heads
130 -
131 - self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
132 - trunc_normal_(self.query, std=0.02)
133 - if kv_dim is not None and kv_dim != embed_dim:
134 - self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
135 - else:
136 - # Maintain the same return value with ReplicatedLinear.forward
137 - self.kv_proj = lambda *args, **kwargs: (
138 - nn.Identity()(*args, **kwargs),
139 - None,
140 - )
141 - self.attn = nn.MultiheadAttention(embed_dim, num_heads)
142 - self.ln_q = norm_layer(embed_dim)
143 - self.ln_kv = norm_layer(embed_dim)
144 - self.ln_post = norm_layer(embed_dim)
145 - self.proj = nn.Parameter(
146 - (embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
147 -
148 - def _init_weights(self, m: nn.Module) -> None:
149 - if isinstance(m, nn.Linear):
150 - trunc_normal_(m.weight, std=0.02)
151 - if isinstance(m, nn.Linear) and m.bias is not None:
152 - nn.init.constant_(m.bias, 0)
153 - elif isinstance(m, nn.LayerNorm):
154 - nn.init.constant_(m.bias, 0)
155 - nn.init.constant_(m.weight, 1.0)
156 -
157 - def _repeat(self, query, N: int):
158 - return query.unsqueeze(1).repeat(1, N, 1)
159 -
160 -
107 class Resampler2_5(BaseResampler):
108
109 def __init__(
@@ -869,7 +815,35 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
815 return "resampler" in name
816
817
872 -class MiniCPMV2_6(MiniCPMVBaseModel):
818 +class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
819 + packed_modules_mapping = {
820 + "qkv_proj": [
821 + "q_proj",
822 + "k_proj",
823 + "v_proj",
824 + ],
825 + "gate_up_proj": [
826 + "gate_proj",
827 + "up_proj",
828 + ],
829 + }
830 + # LoRA specific attributes
831 + supported_lora_modules = [
832 + # vision encoder
833 + "fc1",
834 + "fc2",
835 + "out_proj",
836 + # language model
837 + "qkv_proj", # same name with vision encoder
838 + "o_proj",
839 + "gate_up_proj",
840 + "down_proj",
841 + # resampler
842 + "kv_proj",
843 + ]
844 +
845 + embedding_modules = {}
846 + embedding_padding_modules = []
847
848 def __init__(
849 self,
@@ -894,15 +868,8 @@ class MiniCPMV2_6(MiniCPMVBaseModel):
868 name="model")
869
870 def init_vision_module(self) -> nn.Module:
897 - # A custom version of SiglipVisionTransformer, won't work with TP
898 - from vllm.model_executor.models.na_vit import SiglipVisionTransformer
871
900 - if self.config._attn_implementation == "flash_attention_2":
901 - self.config.vision_config._attn_implementation = "flash_attention_2"
902 - else:
903 - # not support sdpa
904 - self.config.vision_config._attn_implementation = "eager"
905 - model = SiglipVisionTransformer(self.config.vision_config)
872 + model = Idefics2VisionTransformer(self.config.vision_config)
873 if self.config.drop_vision_last_layer:
874 model.encoder.layers = model.encoder.layers[:-1]
875 return model
@@ -928,7 +895,7 @@ class MiniCPMV2_6(MiniCPMVBaseModel):
895 pixel_values,
896 patch_attention_mask=patch_attn_mask,
897 tgt_sizes=tgt_sizes,
931 - ).last_hidden_state
898 + )
899 return vision_embedding
900
901 def get_vision_hidden_states(
@@ -960,12 +927,12 @@ class MiniCPMV2_6(MiniCPMVBaseModel):
927 all_pixel_values.type(dtype),
928 patch_attention_mask=patch_attn_mask,
929 tgt_sizes=tgt_sizes,
963 - ).last_hidden_state
930 + )
931
932 return self.resampler(vision_embedding, tgt_sizes)
933
934 def is_default_weight_loading(self, name: str) -> bool:
968 - return "resampler" in name or "vpm" in name
935 + return "resampler" in name
936
937
938 _SUPPORT_VERSION = {
vllm/model_executor/models/na_vit.py deleted
-804
@@ -1,804 +0,0 @@
1 -import logging
2 -import math
3 -import os
4 -import warnings
5 -from typing import Optional, Tuple, Union
6 -
7 -import numpy as np
8 -import torch
9 -import torch.nn.functional as F
10 -from torch import nn
11 -from torch.nn.init import _calculate_fan_in_and_fan_out
12 -from transformers.activations import ACT2FN
13 -from transformers.configuration_utils import PretrainedConfig
14 -from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
15 -from transformers.modeling_outputs import (BaseModelOutput,
16 - BaseModelOutputWithPooling)
17 -from transformers.modeling_utils import PreTrainedModel
18 -from transformers.utils import (ModelOutput, is_flash_attn_2_available,
19 - replace_return_docstrings)
20 -
21 -logger = logging.getLogger("vllm")
22 -
23 -
24 -# For Siglip: copied from
25 -# HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit and add tgt_sizes
26 -# Remove hints as there's little possibility to change these code.
27 -class SiglipVisionConfig(PretrainedConfig):
28 -
29 - model_type = "siglip_vision_model"
30 -
31 - def __init__(
32 - self,
33 - hidden_size=768,
34 - intermediate_size=3072,
35 - num_hidden_layers=12,
36 - num_attention_heads=12,
37 - num_channels=3,
38 - image_size=224,
39 - patch_size=16,
40 - hidden_act="gelu_pytorch_tanh",
41 - layer_norm_eps=1e-6,
42 - attention_dropout=0.0,
43 - **kwargs,
44 - ):
45 - super().__init__(**kwargs)
46 -
47 - self.hidden_size = hidden_size
48 - self.intermediate_size = intermediate_size
49 - self.num_hidden_layers = num_hidden_layers
50 - self.num_attention_heads = num_attention_heads
51 - self.num_channels = num_channels
52 - self.patch_size = patch_size
53 - self.image_size = image_size
54 - self.attention_dropout = attention_dropout
55 - self.layer_norm_eps = layer_norm_eps
56 - self.hidden_act = hidden_act
57 -
58 - @classmethod
59 - def from_pretrained(cls, pretrained_model_name_or_path: Union[str,
60 - os.PathLike],
61 - **kwargs) -> "PretrainedConfig":
62 - cls._set_token_in_kwargs(kwargs)
63 -
64 - config_dict, kwargs = cls.get_config_dict(
65 - pretrained_model_name_or_path, **kwargs)
66 -
67 - # get the vision config dict if we are loading from SiglipConfig
68 - if config_dict.get("model_type") == "siglip":
69 - config_dict = config_dict["vision_config"]
70 -
71 - if "model_type" in config_dict and hasattr(
72 - cls,
73 - "model_type") and config_dict["model_type"] != cls.model_type:
74 - logger.warning(
75 - "You are using a model of type %s to "
76 - "instantiate a model of type %s. "
77 - "This is not supported for all configurations"
78 - "of models and can yield errors.", config_dict['model_type'],
79 - cls.model_type)
80 -
81 - return cls.from_dict(config_dict, **kwargs)
82 -
83 -
84 -_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
85 -
86 -SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
87 - "google/siglip-base-patch16-224",
88 - # See all SigLIP models at https://huggingface.co/models?filter=siglip
89 -]
90 -
91 -if is_flash_attn_2_available():
92 - from flash_attn import flash_attn_func, flash_attn_varlen_func
93 - from flash_attn.bert_padding import pad_input # noqa
94 - from flash_attn.bert_padding import index_first_axis, unpad_input
95 -
96 -
97 -# Copied from transformers.models.llama.modeling_llama._get_unpad_data
98 -def _get_unpad_data(attention_mask):
99 - seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
100 - indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
101 - max_seqlen_in_batch = seqlens_in_batch.max().item()
102 - cu_seqlens = F.pad(
103 - torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
104 - return (
105 - indices,
106 - cu_seqlens,
107 - max_seqlen_in_batch,
108 - )
109 -
110 -
111 -def _trunc_normal_(tensor, mean, std, a, b):
112 -
113 - def norm_cdf(x):
114 - # Computes standard normal cumulative distribution function
115 - return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
116 -
117 - if (mean < a - 2 * std) or (mean > b + 2 * std):
118 - warnings.warn(
119 - "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
120 - "The distribution of values may be incorrect.",
121 - stacklevel=2,
122 - )
123 -
124 - # Values are generated by using a truncated uniform distribution and
125 - # then using the inverse CDF for the normal distribution.
126 - # Get upper and lower cdf values
127 - l_ = norm_cdf((a - mean) / std)
128 - u = norm_cdf((b - mean) / std)
129 -
130 - # Uniformly fill tensor with values from [l, u], then translate to
131 - # [2l-1, 2u-1].
132 - tensor.uniform_(2 * l_ - 1, 2 * u - 1)
133 -
134 - # Use inverse cdf transform for normal distribution to get truncated
135 - # standard normal
136 - if tensor.dtype in [torch.float16, torch.bfloat16]:
137 - # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
138 - og_dtype = tensor.dtype
139 - tensor = tensor.to(torch.float32)
140 - tensor.erfinv_()
141 - tensor = tensor.to(og_dtype)
142 - else:
143 - tensor.erfinv_()
144 -
145 - # Transform to proper mean, std
146 - tensor.mul_(std * math.sqrt(2.0))
147 - tensor.add_(mean)
148 -
149 - # Clamp to ensure it's in the proper range
150 - if tensor.dtype == torch.float16:
151 - # The `clamp_` op is not (yet?) defined in float16+cpu
152 - tensor = tensor.to(torch.float32)
153 - tensor.clamp_(min=a, max=b)
154 - tensor = tensor.to(torch.float16)
155 - else:
156 - tensor.clamp_(min=a, max=b)
157 -
158 -
159 -def trunc_normal_tf_(tensor: torch.Tensor,
160 - mean: float = 0.0,
161 - std: float = 1.0,
162 - a: float = -2.0,
163 - b: float = 2.0) -> torch.Tensor:
164 - with torch.no_grad():
165 - _trunc_normal_(tensor, 0, 1.0, a, b)
166 - tensor.mul_(std).add_(mean)
167 -
168 -
169 -def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
170 - fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
171 - if mode == "fan_in":
172 - denom = fan_in
173 - elif mode == "fan_out":
174 - denom = fan_out
175 - elif mode == "fan_avg":
176 - denom = (fan_in + fan_out) / 2
177 -
178 - variance = scale / denom
179 -
180 - if distribution == "truncated_normal":
181 - # constant is stddev of standard normal truncated to (-2, 2)
182 - trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
183 - elif distribution == "normal":
184 - with torch.no_grad():
185 - tensor.normal_(std=math.sqrt(variance))
186 - elif distribution == "uniform":
187 - bound = math.sqrt(3 * variance)
188 - with torch.no_grad():
189 - tensor.uniform_(-bound, bound)
190 - else:
191 - raise ValueError(f"invalid distribution {distribution}")
192 -
193 -
194 -def lecun_normal_(tensor):
195 - variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
196 -
197 -
198 -def default_flax_embed_init(tensor):
199 - variance_scaling_(tensor, mode="fan_in", distribution="normal")
200 -
201 -
202 -class SiglipVisionModelOutput(ModelOutput):
203 - image_embeds: Optional[torch.FloatTensor] = None
204 - last_hidden_state: torch.FloatTensor = None
205 - hidden_states: Optional[Tuple[torch.FloatTensor]] = None
206 - attentions: Optional[Tuple[torch.FloatTensor]] = None
207 -
208 -
209 -class SiglipVisionEmbeddings(nn.Module):
210 -
211 - def __init__(self, config: SiglipVisionConfig):
212 - super().__init__()
213 - self.config = config
214 - self.embed_dim = config.hidden_size
215 - self.image_size = config.image_size
216 - self.patch_size = config.patch_size
217 -
218 - self.patch_embedding = nn.Conv2d(
219 - in_channels=config.num_channels,
220 - out_channels=self.embed_dim,
221 - kernel_size=self.patch_size,
222 - stride=self.patch_size,
223 - padding="valid",
224 - )
225 -
226 - self.num_patches_per_side = self.image_size // self.patch_size
227 - self.num_patches = self.num_patches_per_side**2
228 - self.num_positions = self.num_patches
229 - self.position_embedding = nn.Embedding(self.num_positions,
230 - self.embed_dim)
231 -
232 - def forward(self,
233 - pixel_values: torch.FloatTensor,
234 - patch_attention_mask: torch.BoolTensor,
235 - tgt_sizes: Optional[torch.IntTensor] = None) -> torch.Tensor:
236 - batch_size = pixel_values.size(0)
237 -
238 - patch_embeds = self.patch_embedding(pixel_values)
239 - embeddings = patch_embeds.flatten(2).transpose(1, 2)
240 -
241 - max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3)
242 - max_nb_patches_h, max_nb_patches_w = (max_im_h // self.patch_size,
243 - max_im_w // self.patch_size)
244 - boundaries = torch.arange(1 / self.num_patches_per_side, 1.0,
245 - 1 / self.num_patches_per_side)
246 - position_ids = torch.full(
247 - size=(
248 - batch_size,
249 - max_nb_patches_h * max_nb_patches_w,
250 - ),
251 - fill_value=0,
252 - )
253 -
254 - for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
255 - if tgt_sizes is not None:
256 - nb_patches_h = tgt_sizes[batch_idx][0]
257 - nb_patches_w = tgt_sizes[batch_idx][1]
258 - else:
259 - nb_patches_h = p_attn_mask[:, 0].sum()
260 - nb_patches_w = p_attn_mask[0].sum()
261 -
262 - fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
263 - fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
264 -
265 - bucket_coords_h = torch.bucketize(fractional_coords_h,
266 - boundaries,
267 - right=True)
268 - bucket_coords_w = torch.bucketize(fractional_coords_w,
269 - boundaries,
270 - right=True)
271 -
272 - pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side +
273 - bucket_coords_w).flatten()
274 - position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
275 -
276 - position_ids = position_ids.to(self.position_embedding.weight.device)
277 -
278 - embeddings = embeddings + self.position_embedding(position_ids)
279 - return embeddings
280 -
281 -
282 -class SiglipAttention(nn.Module):
283 - """Multi-headed attention from 'Attention Is All You Need' paper"""
284 -
285 - # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
286 - def __init__(self, config):
287 - super().__init__()
288 - self.config = config
289 - self.embed_dim = config.hidden_size
290 - self.num_heads = config.num_attention_heads
291 - self.head_dim = self.embed_dim // self.num_heads
292 - if self.head_dim * self.num_heads != self.embed_dim:
293 - raise ValueError(
294 - "embed_dim must be divisible by num_heads (got `embed_dim`: "
295 - f"{self.embed_dim} and `num_heads`:"
296 - f" {self.num_heads}).")
297 - self.scale = self.head_dim**-0.5
298 - self.dropout = config.attention_dropout
299 -
300 - self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
301 - self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
302 - self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
303 - self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
304 -
305 - def forward(
306 - self,
307 - hidden_states: torch.Tensor,
308 - attention_mask: Optional[torch.Tensor] = None,
309 - output_attentions: Optional[bool] = False,
310 - ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
311 - Optional[Tuple[torch.Tensor]]]:
312 - """Input shape: Batch x Time x Channel"""
313 -
314 - batch_size, q_len, _ = hidden_states.size()
315 -
316 - query_states = self.q_proj(hidden_states)
317 - key_states = self.k_proj(hidden_states)
318 - value_states = self.v_proj(hidden_states)
319 -
320 - query_states = query_states.view(batch_size, q_len, self.num_heads,
321 - self.head_dim).transpose(1, 2)
322 - key_states = key_states.view(batch_size, q_len, self.num_heads,
323 - self.head_dim).transpose(1, 2)
324 - value_states = value_states.view(batch_size, q_len, self.num_heads,
325 - self.head_dim).transpose(1, 2)
326 -
327 - k_v_seq_len = key_states.shape[-2]
328 - attn_weights = torch.matmul(query_states, key_states.transpose(
329 - 2, 3)) * self.scale
330 -
331 - if attn_weights.size() != (batch_size, self.num_heads, q_len,
332 - k_v_seq_len):
333 - raise ValueError(
334 - "Attention weights should be of size "
335 - f"{(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
336 - f" {attn_weights.size()}")
337 -
338 - if attention_mask is not None:
339 - if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
340 - raise ValueError(
341 - "Attention mask should be of size "
342 - f"{(batch_size, 1, q_len, k_v_seq_len)}",
343 - f"but is {attention_mask.size()}")
344 - attn_weights = attn_weights + attention_mask
345 -
346 - # upcast attention to fp32
347 - attn_weights = nn.functional.softmax(attn_weights,
348 - dim=-1,
349 - dtype=torch.float32).to(
350 - query_states.dtype)
351 - attn_weights = nn.functional.dropout(attn_weights,
352 - p=self.dropout,
353 - training=self.training)
354 - attn_output = torch.matmul(attn_weights, value_states)
355 -
356 - if attn_output.size() != (batch_size, self.num_heads, q_len,
357 - self.head_dim):
358 - raise ValueError(
359 - "`attn_output` should be of size "
360 - f"{(batch_size, self.num_heads, q_len, self.head_dim)}, "
361 - "but is"
362 - f" {attn_output.size()}")
363 -
364 - attn_output = attn_output.transpose(1, 2).contiguous()
365 - attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
366 -
367 - attn_output = self.out_proj(attn_output)
368 -
369 - return attn_output, attn_weights
370 -
371 -
372 -class SiglipFlashAttention2(SiglipAttention):
373 -
374 - def __init__(self, *args, **kwargs):
375 - super().__init__(*args, **kwargs)
376 - self.is_causal = False # Hack to make sure we don't use a causal mask
377 -
378 - def forward(
379 - self,
380 - hidden_states: torch.Tensor,
381 - attention_mask: Optional[torch.LongTensor] = None,
382 - position_ids: Optional[torch.LongTensor] = None,
383 - past_key_value: Optional[Tuple[torch.Tensor]] = None,
384 - output_attentions: bool = False,
385 - use_cache: bool = False,
386 - **kwargs,
387 - ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
388 - Optional[Tuple[torch.Tensor]]]:
389 - output_attentions = False
390 -
391 - bsz, q_len, _ = hidden_states.size()
392 -
393 - query_states = self.q_proj(hidden_states)
394 - key_states = self.k_proj(hidden_states)
395 - value_states = self.v_proj(hidden_states)
396 -
397 - query_states = query_states.view(bsz, q_len, self.num_heads,
398 - self.head_dim).transpose(1, 2)
399 - key_states = key_states.view(bsz, q_len, self.num_heads,
400 - self.head_dim).transpose(1, 2)
401 - value_states = value_states.view(bsz, q_len, self.num_heads,
402 - self.head_dim).transpose(1, 2)
403 -
404 - kv_seq_len = key_states.shape[-2]
405 - if past_key_value is not None:
406 - kv_seq_len += past_key_value.get_usable_length(
407 - kv_seq_len, self.layer_idx)
408 -
409 - query_states = query_states.transpose(1, 2)
410 - key_states = key_states.transpose(1, 2)
411 - value_states = value_states.transpose(1, 2)
412 -
413 - dropout_rate = self.dropout if self.training else 0.0
414 -
415 - input_dtype = query_states.dtype
416 - if input_dtype == torch.float32:
417 - if torch.is_autocast_enabled():
418 - target_dtype = torch.get_autocast_gpu_dtype()
419 - # Handle the case where the model is quantized
420 - elif hasattr(self.config, "_pre_quantization_dtype"):
421 - target_dtype = self.config._pre_quantization_dtype
422 - else:
423 - target_dtype = self.q_proj.weight.dtype
424 -
425 - logger.warning(
426 - "The input hidden states seems to be "
427 - "silently casted in float32, "
428 - "this might be related to the fact "
429 - "you have upcasted embedding or layer norm layers in float32. "
430 - "We will cast back the input in"
431 - " %s.", target_dtype)
432 -
433 - query_states = query_states.to(target_dtype)
434 - key_states = key_states.to(target_dtype)
435 - value_states = value_states.to(target_dtype)
436 -
437 - attn_output = self._flash_attention_forward(query_states,
438 - key_states,
439 - value_states,
440 - attention_mask,
441 - q_len,
442 - dropout=dropout_rate)
443 -
444 - attn_output = attn_output.reshape(bsz, q_len,
445 - self.embed_dim).contiguous()
446 - attn_output = self.out_proj(attn_output)
447 -
448 - if not output_attentions:
449 - attn_weights = None
450 -
451 - return attn_output, attn_weights
452 -
453 - def _flash_attention_forward(self,
454 - query_states,
455 - key_states,
456 - value_states,
457 - attention_mask,
458 - query_length,
459 - dropout=0.0,
460 - softmax_scale=None):
461 - causal = self.is_causal and query_length != 1
462 -
463 - # Contains at least one padding token in the sequence
464 - if attention_mask is not None:
465 - batch_size = query_states.shape[0]
466 - (query_states, key_states, value_states, indices_q, cu_seq_lens,
467 - max_seq_lens) = self._upad_input(query_states, key_states,
468 - value_states, attention_mask,
469 - query_length)
470 -
471 - cu_seqlens_q, cu_seqlens_k = cu_seq_lens
472 - max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
473 -
474 - attn_output_unpad = flash_attn_varlen_func(
475 - query_states,
476 - key_states,
477 - value_states,
478 - cu_seqlens_q=cu_seqlens_q,
479 - cu_seqlens_k=cu_seqlens_k,
480 - max_seqlen_q=max_seqlen_in_batch_q,
481 - max_seqlen_k=max_seqlen_in_batch_k,
482 - dropout_p=dropout,
483 - softmax_scale=softmax_scale,
484 - causal=causal,
485 - )
486 -
487 - attn_output = pad_input(attn_output_unpad, indices_q, batch_size,
488 - query_length)
489 - else:
490 - attn_output = flash_attn_func(query_states,
491 - key_states,
492 - value_states,
493 - dropout,
494 - softmax_scale=softmax_scale,
495 - causal=causal)
496 -
497 - return attn_output
498 -
499 - def _upad_input(self, query_layer, key_layer, value_layer, attention_mask,
500 - query_length):
501 - indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(
502 - attention_mask)
503 - batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
504 -
505 - key_layer = index_first_axis(
506 - key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
507 - head_dim), indices_k)
508 - value_layer = index_first_axis(
509 - value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
510 - head_dim), indices_k)
511 - if query_length == kv_seq_len:
512 - query_layer = index_first_axis(
513 - query_layer.reshape(batch_size * kv_seq_len, self.num_heads,
514 - head_dim), indices_k)
515 - cu_seqlens_q = cu_seqlens_k
516 - max_seqlen_in_batch_q = max_seqlen_in_batch_k
517 - indices_q = indices_k
518 - elif query_length == 1:
519 - max_seqlen_in_batch_q = 1
520 - cu_seqlens_q = torch.arange(
521 - batch_size + 1, dtype=torch.int32, device=query_layer.device
522 - ) # There is a memcpy here, that is very bad.
523 - indices_q = cu_seqlens_q[:-1]
524 - query_layer = query_layer.squeeze(1)
525 - else:
526 - # The -q_len: slice assumes left padding.
527 - attention_mask = attention_mask[:, -query_length:]
528 - (query_layer, indices_q, cu_seqlens_q,
529 - max_seqlen_in_batch_q) = unpad_input(query_layer, attention_mask)
530 -
531 - return (
532 - query_layer,
533 - key_layer,
534 - value_layer,
535 - indices_q,
536 - (cu_seqlens_q, cu_seqlens_k),
537 - (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
538 - )
539 -
540 -
541 -# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
542 -class SiglipMLP(nn.Module):
543 -
544 - def __init__(self, config):
545 - super().__init__()
546 - self.config = config
547 - self.activation_fn = ACT2FN[config.hidden_act]
548 - self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
549 - self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
550 -
551 - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
552 - hidden_states = self.fc1(hidden_states)
553 - hidden_states = self.activation_fn(hidden_states)
554 - hidden_states = self.fc2(hidden_states)
555 - return hidden_states
556 -
557 -
558 -# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer
559 -# with CLIP->Siglip
560 -class SiglipEncoderLayer(nn.Module):
561 -
562 - def __init__(self, config: SiglipVisionConfig):
563 - super().__init__()
564 - self.embed_dim = config.hidden_size
565 - self._use_flash_attention_2 = (
566 - config._attn_implementation == "flash_attention_2")
567 - self.self_attn = (SiglipAttention(config)
568 - if not self._use_flash_attention_2 else
569 - SiglipFlashAttention2(config))
570 - self.layer_norm1 = nn.LayerNorm(self.embed_dim,
571 - eps=config.layer_norm_eps)
572 - self.mlp = SiglipMLP(config)
573 - self.layer_norm2 = nn.LayerNorm(self.embed_dim,
574 - eps=config.layer_norm_eps)
575 -
576 - def forward(
577 - self,
578 - hidden_states: torch.Tensor,
579 - attention_mask: torch.Tensor,
580 - output_attentions: Optional[bool] = False,
581 - ) -> Tuple[torch.FloatTensor]:
582 - residual = hidden_states
583 -
584 - hidden_states = self.layer_norm1(hidden_states)
585 - hidden_states, attn_weights = self.self_attn(
586 - hidden_states=hidden_states,
587 - attention_mask=attention_mask,
588 - output_attentions=output_attentions,
589 - )
590 - hidden_states = residual + hidden_states
591 -
592 - residual = hidden_states
593 - hidden_states = self.layer_norm2(hidden_states)
594 - hidden_states = self.mlp(hidden_states)
595 - hidden_states = residual + hidden_states
596 -
597 - outputs = (hidden_states, )
598 -
599 - if output_attentions:
600 - outputs += (attn_weights, )
601 -
602 - return outputs
603 -
604 -
605 -class SiglipPreTrainedModel(PreTrainedModel):
606 - config_class = SiglipVisionConfig
607 - base_model_prefix = "siglip"
608 - supports_gradient_checkpointing = True
609 -
610 - def _init_weights(self, module):
611 - """Initialize the weights"""
612 -
613 - if isinstance(module, SiglipVisionEmbeddings):
614 - width = self.config.hidden_size
615 - nn.init.normal_(module.position_embedding.weight,
616 - std=1 / np.sqrt(width))
617 - elif isinstance(module, nn.Embedding):
618 - default_flax_embed_init(module.weight)
619 - elif isinstance(module, SiglipAttention):
620 - nn.init.normal_(module.q_proj.weight)
621 - nn.init.normal_(module.k_proj.weight)
622 - nn.init.normal_(module.v_proj.weight)
623 - nn.init.normal_(module.out_proj.weight)
624 - nn.init.zeros_(module.q_proj.bias)
625 - nn.init.zeros_(module.k_proj.bias)
626 - nn.init.zeros_(module.v_proj.bias)
627 - nn.init.zeros_(module.out_proj.bias)
628 - elif isinstance(module, SiglipMLP):
629 - nn.init.normal_(module.fc1.weight)
630 - nn.init.normal_(module.fc2.weight)
631 - nn.init.normal_(module.fc1.bias, std=1e-6)
632 - nn.init.normal_(module.fc2.bias, std=1e-6)
633 - elif isinstance(module, (nn.Linear, nn.Conv2d)):
634 - lecun_normal_(module.weight)
635 - if module.bias is not None:
636 - nn.init.zeros_(module.bias)
637 - elif isinstance(module, nn.LayerNorm):
638 - module.bias.data.zero_()
639 - module.weight.data.fill_(1.0)
640 -
641 -
642 -# Copied from transformers.models.clip.modeling_clip.CLIPEncoder
643 -# with CLIP->Siglip
644 -class SiglipEncoder(nn.Module):
645 -
646 - def __init__(self, config: SiglipVisionConfig):
647 - super().__init__()
648 - self.config = config
649 - self.layers = nn.ModuleList([
650 - SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)
651 - ])
652 - self.gradient_checkpointing = False
653 -
654 - # Ignore copy
655 - def forward(
656 - self,
657 - inputs_embeds,
658 - attention_mask: Optional[torch.Tensor] = None,
659 - output_attentions: Optional[bool] = None,
660 - output_hidden_states: Optional[bool] = None,
661 - return_dict: Optional[bool] = None,
662 - ) -> Union[Tuple, BaseModelOutput]:
663 - output_attentions = output_attentions if output_attentions is not None \
664 - else self.config.output_attentions
665 - output_hidden_states = (output_hidden_states
666 - if output_hidden_states is not None else
667 - self.config.output_hidden_states)
668 - return_dict = return_dict if return_dict is not None \
669 - else self.config.use_return_dict
670 -
671 - encoder_states = () if output_hidden_states else None
672 - all_attentions = () if output_attentions else None
673 -
674 - hidden_states = inputs_embeds
675 - for encoder_layer in self.layers:
676 - if output_hidden_states:
677 - encoder_states = encoder_states + (hidden_states, )
678 - if self.gradient_checkpointing and self.training:
679 - layer_outputs = self._gradient_checkpointing_func(
680 - encoder_layer.__call__,
681 - hidden_states,
682 - attention_mask,
683 - output_attentions,
684 - )
685 - else:
686 - layer_outputs = encoder_layer(
687 - hidden_states,
688 - attention_mask,
689 - output_attentions=output_attentions,
690 - )
691 -
692 - hidden_states = layer_outputs[0]
693 -
694 - if output_attentions:
695 - all_attentions = all_attentions + (layer_outputs[1], )
696 -
697 - if output_hidden_states:
698 - encoder_states = encoder_states + (hidden_states, )
699 -
700 - if not return_dict:
701 - return tuple(
702 - v for v in [hidden_states, encoder_states, all_attentions]
703 - if v is not None)
704 - return BaseModelOutput(last_hidden_state=hidden_states,
705 - hidden_states=encoder_states,
706 - attentions=all_attentions)
707 -
708 -
709 -class SiglipVisionTransformer(SiglipPreTrainedModel):
710 - config_class = SiglipVisionConfig
711 - main_input_name = "pixel_values"
712 - _supports_flash_attn_2 = True
713 -
714 - def __init__(self, config: SiglipVisionConfig):
715 - super().__init__(config)
716 - self.config = config
717 - embed_dim = config.hidden_size
718 -
719 - self.embeddings = SiglipVisionEmbeddings(config)
720 - self.encoder = SiglipEncoder(config)
721 - self.post_layernorm = nn.LayerNorm(embed_dim,
722 - eps=config.layer_norm_eps)
723 - self._use_flash_attention_2 = (
724 - config._attn_implementation == "flash_attention_2")
725 -
726 - # Initialize weights and apply final processing
727 - self.post_init()
728 -
729 - def get_input_embeddings(self) -> nn.Module:
730 - return self.embeddings.patch_embedding
731 -
732 - @replace_return_docstrings(output_type=BaseModelOutputWithPooling,
733 - config_class=SiglipVisionConfig)
734 - def forward(
735 - self,
736 - pixel_values,
737 - patch_attention_mask: Optional[torch.BoolTensor] = None,
738 - tgt_sizes: Optional[torch.IntTensor] = None,
739 - output_attentions: Optional[bool] = None,
740 - output_hidden_states: Optional[bool] = None,
741 - return_dict: Optional[bool] = None,
742 - ) -> Union[Tuple, BaseModelOutputWithPooling]:
743 - r"""
744 - Returns:
745 - """
746 - output_attentions = output_attentions if output_attentions is not None \
747 - else self.config.output_attentions
748 - output_hidden_states = (output_hidden_states
749 - if output_hidden_states is not None else
750 - self.config.output_hidden_states)
751 - return_dict = return_dict if return_dict is not None \
752 - else self.config.use_return_dict
753 -
754 - batch_size = pixel_values.size(0)
755 - if patch_attention_mask is None:
756 - patch_attention_mask = torch.ones(
757 - size=(
758 - batch_size,
759 - pixel_values.size(2) // self.config.patch_size,
760 - pixel_values.size(3) // self.config.patch_size,
761 - ),
762 - dtype=torch.bool,
763 - device=pixel_values.device,
764 - )
765 -
766 - hidden_states = self.embeddings(
767 - pixel_values=pixel_values,
768 - patch_attention_mask=patch_attention_mask,
769 - tgt_sizes=tgt_sizes)
770 -
771 - patch_attention_mask = patch_attention_mask.view(batch_size, -1)
772 - # The call to `_upad_input` in `_flash_attention_forward` is expensive
773 - # So when the `patch_attention_mask` is full of 1s
774 - # (i.e. attending to the whole sequence),
775 - # avoiding passing the attention_mask,
776 - # which is equivalent to attending to the full sequence
777 - if not torch.any(~patch_attention_mask):
778 - attention_mask = None
779 - else:
780 - attention_mask = (_prepare_4d_attention_mask(
781 - patch_attention_mask, hidden_states.dtype)
782 - if not self._use_flash_attention_2 else
783 - patch_attention_mask)
784 -
785 - encoder_outputs = self.encoder(
786 - inputs_embeds=hidden_states,
787 - attention_mask=attention_mask,
788 - output_attentions=output_attentions,
789 - output_hidden_states=output_hidden_states,
790 - return_dict=return_dict,
791 - )
792 -
793 - last_hidden_state = encoder_outputs[0]
794 - last_hidden_state = self.post_layernorm(last_hidden_state)
795 -
796 - if not return_dict:
797 - return (last_hidden_state, None) + encoder_outputs[1:]
798 -
799 - return BaseModelOutputWithPooling(
800 - last_hidden_state=last_hidden_state,
801 - pooler_output=None,
802 - hidden_states=encoder_outputs.hidden_states,
803 - attentions=encoder_outputs.attentions,
804 - )