@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 10398b4706
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Consolidate ViTs attention implementation without mask (#10893)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Dec 5, 2024 at 02:11 UTC 10398b4706ee71d0bddc32c1d33b11e73df12a27
9 files changed +109 -226
vllm/attention/layer.py
+63
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional
3
4 import torch
5 import torch.nn as nn
6 +import torch.nn.functional as F
7
8 from vllm.attention import AttentionMetadata, AttentionType
9 from vllm.attention.selector import backend_name_to_enum, get_attn_backend
@@ -168,6 +169,68 @@ class Attention(nn.Module):
169 return s
170
171
172 +class MultiHeadAttention(nn.Module):
173 + """Multi-headed attention without any cache, used for ViT."""
174 +
175 + def __init__(
176 + self,
177 + num_heads: int,
178 + head_size: int,
179 + scale: float,
180 + num_kv_heads: Optional[int] = None,
181 + ):
182 + super().__init__()
183 + self.num_heads = num_heads
184 + self.head_size = head_size
185 + self.scale = scale
186 + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
187 +
188 + dtype = torch.get_default_dtype()
189 + attn_backend = get_attn_backend(head_size,
190 + dtype,
191 + kv_cache_dtype=None,
192 + block_size=16,
193 + is_attention_free=False)
194 + if attn_backend in {_Backend.FLASH_ATTN, _Backend.FLASH_ATTN_VLLM_V1}:
195 + attn_backend = _Backend.XFORMERS
196 +
197 + self.attn_backend = attn_backend if attn_backend in {
198 + _Backend.TORCH_SDPA, _Backend.XFORMERS
199 + } else _Backend.TORCH_SDPA
200 +
201 + def forward(
202 + self,
203 + query: torch.Tensor,
204 + key: torch.Tensor,
205 + value: torch.Tensor,
206 + ) -> torch.Tensor:
207 + """Input shape: batch_size x seq_len x hidden_size"""
208 + # TODO(Isotr0py): Use existing backend implementations and support FA2
209 + bsz, q_len, _ = query.size()
210 + kv_len = key.size(1)
211 +
212 + query = query.view(bsz, q_len, self.num_heads, self.head_size)
213 + key = key.view(bsz, kv_len, self.num_kv_heads, self.head_size)
214 + value = value.view(bsz, kv_len, self.num_kv_heads, self.head_size)
215 +
216 + if self.attn_backend == _Backend.XFORMERS:
217 + from xformers import ops as xops
218 +
219 + out = xops.memory_efficient_attention_forward(query,
220 + key,
221 + value,
222 + scale=self.scale)
223 + elif self.attn_backend == _Backend.TORCH_SDPA:
224 + query, key, value = (x.transpose(1, 2)
225 + for x in (query, key, value))
226 + out = F.scaled_dot_product_attention(query,
227 + key,
228 + value,
229 + scale=self.scale)
230 + out = out.transpose(1, 2)
231 + return out.view(bsz, q_len, -1)
232 +
233 +
234 def unified_attention(
235 query: torch.Tensor,
236 key: torch.Tensor,
vllm/model_executor/models/blip.py
+4 -41
@@ -4,11 +4,10 @@ from typing import Iterable, Optional, Set, Tuple, Union
4
5 import torch
6 import torch.nn as nn
7 -import torch.nn.functional as F
7 from PIL import Image
8 from transformers import Blip2VisionConfig, BlipVisionConfig
9
11 -from vllm.attention.selector import _Backend
10 +from vllm.attention.layer import MultiHeadAttention
11 from vllm.config import ModelConfig
12 from vllm.distributed import divide, get_tensor_model_parallel_world_size
13 from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -22,8 +21,6 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
21 repeat_and_pad_placeholder_tokens)
22 from vllm.sequence import SequenceData
23
25 -from .utils import get_vit_attn_backend
26 -
24
25 def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
26 assert image_size % patch_size == 0
@@ -205,11 +202,8 @@ class BlipAttention(nn.Module):
202 self.tp_size = get_tensor_model_parallel_world_size()
203 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
204
208 - # Detect attention implementation.
209 - self.attn_backend = get_vit_attn_backend(support_fa=False)
210 - if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
211 - raise RuntimeError(
212 - f"BLIP does not support {self.attn_backend} backend now.")
205 + self.attn = MultiHeadAttention(self.num_heads_per_partition,
206 + self.head_dim, self.scale)
207
208 def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
209 return tensor.view(bsz, seq_len, self.num_heads,
@@ -220,41 +214,10 @@ class BlipAttention(nn.Module):
214 hidden_states: torch.Tensor,
215 ):
216 """Input shape: Batch x Time x Channel"""
223 - bsz, tgt_len, _ = hidden_states.size()
217
218 qkv_states, _ = self.qkv(hidden_states)
219 query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
227 - query_states = query_states.view(bsz, tgt_len,
228 - self.num_heads_per_partition,
229 - self.head_dim)
230 - key_states = key_states.view(bsz, tgt_len,
231 - self.num_heads_per_partition,
232 - self.head_dim)
233 - value_states = value_states.view(bsz, tgt_len,
234 - self.num_heads_per_partition,
235 - self.head_dim)
236 -
237 - if self.attn_backend == _Backend.XFORMERS:
238 - from xformers import ops as xops
239 -
240 - out = xops.memory_efficient_attention_forward(query_states,
241 - key_states,
242 - value_states,
243 - p=self.dropout,
244 - scale=self.scale)
245 - elif self.attn_backend == _Backend.TORCH_SDPA:
246 - query_states, key_states, value_states = (x.transpose(1, 2)
247 - for x in (query_states,
248 - key_states,
249 - value_states))
250 - out = F.scaled_dot_product_attention(query_states,
251 - key_states,
252 - value_states,
253 - dropout_p=self.dropout,
254 - scale=self.scale)
255 - out = out.transpose(1, 2)
256 -
257 - out = out.view(bsz, tgt_len, -1)
220 + out = self.attn(query_states, key_states, value_states)
221 attn_output, _ = self.projection(out)
222
223 return attn_output, None
vllm/model_executor/models/clip.py
+4 -42
@@ -5,11 +5,10 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
5 import numpy as np
6 import torch
7 import torch.nn as nn
8 -import torch.nn.functional as F
8 from PIL import Image
9 from transformers import CLIPVisionConfig
10
12 -from vllm.attention.selector import _Backend
11 +from vllm.attention.layer import MultiHeadAttention
12 from vllm.config import ModelConfig
13 from vllm.distributed import divide, get_tensor_model_parallel_world_size
14 from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -25,8 +24,6 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
24 resolve_visual_encoder_outputs)
25 from vllm.sequence import SequenceData
26
28 -from .utils import get_vit_attn_backend
29 -
27
28 def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
29 assert image_size % patch_size == 0
@@ -235,11 +232,8 @@ class CLIPAttention(nn.Module):
232 self.tp_size = get_tensor_model_parallel_world_size()
233 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
234
238 - # Detect attention implementation.
239 - self.attn_backend = get_vit_attn_backend(support_fa=False)
240 - if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
241 - raise RuntimeError(
242 - f"CLIP does not support {self.attn_backend} backend now.")
235 + self.attn = MultiHeadAttention(self.num_heads_per_partition,
236 + self.head_dim, self.scale)
237
238 def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
239 return tensor.view(bsz, seq_len, self.num_heads,
@@ -250,42 +244,10 @@ class CLIPAttention(nn.Module):
244 hidden_states: torch.Tensor,
245 ):
246 """Input shape: Batch x Time x Channel"""
253 - bsz, tgt_len, _ = hidden_states.size()
247
248 qkv_states, _ = self.qkv_proj(hidden_states)
249 query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
257 -
258 - query_states = query_states.view(bsz, tgt_len,
259 - self.num_heads_per_partition,
260 - self.head_dim)
261 - key_states = key_states.view(bsz, tgt_len,
262 - self.num_heads_per_partition,
263 - self.head_dim)
264 - value_states = value_states.view(bsz, tgt_len,
265 - self.num_heads_per_partition,
266 - self.head_dim)
267 -
268 - if self.attn_backend == _Backend.XFORMERS:
269 - from xformers import ops as xops
270 -
271 - out = xops.memory_efficient_attention_forward(query_states,
272 - key_states,
273 - value_states,
274 - p=self.dropout,
275 - scale=self.scale)
276 - elif self.attn_backend == _Backend.TORCH_SDPA:
277 - query_states, key_states, value_states = (x.transpose(1, 2)
278 - for x in (query_states,
279 - key_states,
280 - value_states))
281 - out = F.scaled_dot_product_attention(query_states,
282 - key_states,
283 - value_states,
284 - dropout_p=self.dropout,
285 - scale=self.scale)
286 - out = out.transpose(1, 2)
287 -
288 - out = out.view(bsz, tgt_len, -1)
250 + out = self.attn(query_states, key_states, value_states)
251 attn_output, _ = self.out_proj(out)
252
253 return attn_output, None
vllm/model_executor/models/glm4_vision_encoder.py
+6 -16
@@ -8,6 +8,7 @@ import torch
8 from torch import nn
9 from torch.nn import LayerNorm
10
11 +from vllm.attention.layer import MultiHeadAttention
12 from vllm.distributed import get_tensor_model_parallel_world_size
13 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
14 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -77,27 +78,16 @@ class Attention(nn.Module):
78 quant_config=quant_config,
79 )
80
81 + self.attn = MultiHeadAttention(self.num_heads_per_rank, self.head_dim,
82 + self.scale)
83 self.output_dropout = torch.nn.Dropout(config.dropout_prob)
84
85 def forward(self, x: torch.Tensor) -> torch.Tensor:
83 - B, L, _ = x.shape
86 qkv, _ = self.query_key_value(x) # B, L, 3 * H * D
87 q, k, v = qkv.chunk(3, dim=-1)
86 - q = q.reshape(B, L, self.num_heads_per_rank,
87 - self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
88 - k = k.reshape(B, L, self.num_heads_per_rank,
89 - self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
90 - v = v.reshape(B, L, self.num_heads_per_rank,
91 - self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
92 -
93 - out = torch.nn.functional.scaled_dot_product_attention(q,
94 - k,
95 - v,
96 - attn_mask=None,
97 - dropout_p=0.,
98 - is_causal=False)
99 -
100 - output, _ = self.dense(out.transpose(1, 2).view(B, L, -1))
88 +
89 + out = self.attn(q, k, v)
90 + output, _ = self.dense(out)
91 output = self.output_dropout(output)
92 return output
93
vllm/model_executor/models/idefics2_vision_model.py
+4 -21
@@ -21,8 +21,8 @@ import torch
21 from torch import nn
22 from transformers.models.idefics2.configuration_idefics2 import (
23 Idefics2Config, Idefics2VisionConfig)
24 -from xformers import ops as xops
24
25 +from vllm.attention.layer import MultiHeadAttention
26 from vllm.distributed import divide, get_tensor_model_parallel_world_size
27 from vllm.model_executor.layers.activation import get_act_fn
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -141,35 +141,18 @@ class Idefics2VisionAttention(nn.Module):
141 )
142 self.tp_size = get_tensor_model_parallel_world_size()
143 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
144 - self.is_causal = False
144 + self.attn = MultiHeadAttention(self.num_heads_per_partition,
145 + self.head_dim, self.scale)
146
147 def forward(
148 self,
149 hidden_states: torch.Tensor,
150 ) -> torch.Tensor:
150 - batch_size, q_len, _ = hidden_states.size()
151 qkv, _ = self.qkv_proj(
152 hidden_states
153 ) # batch_size, q_len, 3 * num_heads_per_partition * head_dim
154 query_states, key_states, value_states = qkv.chunk(3, dim=-1)
155 - query_states = query_states.view(batch_size, q_len,
156 - self.num_heads_per_partition,
157 - self.head_dim)
158 - key_states = key_states.view(batch_size, q_len,
159 - self.num_heads_per_partition,
160 - self.head_dim)
161 - value_states = value_states.view(batch_size, q_len,
162 - self.num_heads_per_partition,
163 - self.head_dim)
164 - # see: https://facebookresearch.github.io/xformers/components/ops.html
165 - out = xops.memory_efficient_attention_forward(
166 - query_states,
167 - key_states,
168 - value_states,
169 - p=self.dropout,
170 - scale=self.scale,
171 - )
172 - out = out.view(batch_size, q_len, -1)
155 + out = self.attn(query_states, key_states, value_states)
156 attn_output, _ = self.out_proj(out)
157 return attn_output
158
vllm/model_executor/models/intern_vit.py
+4 -24
@@ -12,7 +12,7 @@ import torch.nn as nn
12 import torch.nn.functional as F
13 from transformers import PretrainedConfig
14
15 -from vllm.attention.selector import _Backend
15 +from vllm.attention.layer import MultiHeadAttention
16 from vllm.distributed import (divide, get_tensor_model_parallel_rank,
17 get_tensor_model_parallel_world_size,
18 split_tensor_along_last_dim,
@@ -25,8 +25,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
25 from vllm.model_executor.layers.quantization import QuantizationConfig
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
28 -from .utils import get_vit_attn_backend
29 -
28 NORM2FN = {
29 'rms_norm': RMSNorm,
30 'layer_norm': nn.LayerNorm,
@@ -183,10 +181,8 @@ class InternParallelAttention(nn.Module):
181 prefix=f"{prefix}.proj",
182 )
183
186 - self.attn_backend = get_vit_attn_backend(support_fa=False)
187 - if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
188 - raise RuntimeError(
189 - f"InternViT does not support {self.attn_backend} backend now.")
184 + self.attn = MultiHeadAttention(self.num_heads_per_partition,
185 + self.head_dim, self.scale)
186
187 def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
188 if self.tp_size > 1:
@@ -209,23 +205,7 @@ class InternParallelAttention(nn.Module):
205 if self.qk_normalization:
206 q, k = self._apply_qk_norm(q, k)
207
212 - q = q.view(B, N, self.num_heads_per_partition, self.head_dim)
213 - k = k.view(B, N, self.num_heads_per_partition, self.head_dim)
214 - v = v.view(B, N, self.num_heads_per_partition, self.head_dim)
215 -
216 - if self.attn_backend == _Backend.XFORMERS:
217 - from xformers import ops as xops
218 -
219 - out = xops.memory_efficient_attention_forward(q,
220 - k,
221 - v,
222 - scale=self.scale)
223 - elif self.attn_backend == _Backend.TORCH_SDPA:
224 - q, k, v = (x.transpose(1, 2) for x in (q, k, v))
225 - out = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
226 - out = out.transpose(1, 2)
227 -
228 - out = out.view(B, N, -1)
208 + out = self.attn(q, k, v)
209 out, _ = self.proj(out)
210 return out
211
vllm/model_executor/models/internvl.py
+11 -12
@@ -482,6 +482,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
482 self.mlp1 = self._init_mlp1(config)
483
484 self.img_context_token_id = None
485 + self.visual_token_mask = None
486 self.make_empty_intermediate_tensors = (
487 self.language_model.make_empty_intermediate_tensors)
488
@@ -635,13 +636,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
636
637 return image_embeds
638
638 - def _get_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
639 + def _set_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
640 if self.is_mono:
640 - visual_token_mask = (
641 + self.visual_token_mask = (
642 input_ids == self.img_context_token_id).reshape(-1, 1)
643 else:
643 - visual_token_mask = None
644 - return visual_token_mask
644 + self.visual_token_mask = None
645
646 def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
647 image_input = self._parse_and_validate_image_input(**kwargs)
@@ -658,6 +658,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
658 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
659 if multimodal_embeddings is not None:
660 assert self.img_context_token_id is not None
661 + self._set_visual_token_mask(input_ids)
662 inputs_embeds = merge_multimodal_embeddings(
663 input_ids, inputs_embeds, multimodal_embeddings,
664 self.img_context_token_id)
@@ -674,7 +675,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
675 **kwargs: object,
676 ) -> Union[SamplerOutput, IntermediateTensors]:
677
677 - visual_token_mask = None
678 if intermediate_tensors is not None:
679 input_ids = None
680 inputs_embeds = None
@@ -695,16 +695,15 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
695 "intermediate_tensors": intermediate_tensors,
696 "inputs_embeds": inputs_embeds,
697 }
698 - if self.img_context_token_id is not None:
699 - visual_token_mask = self._get_visual_token_mask(input_ids)
698
701 - # We always overwrite it back to None after computing visual token
702 - # mask so that this doesn't need to depend on encoder output
699 + if self.visual_token_mask is not None:
700 + # overwrite visual_token_mask and img_context_token_id back to None,
701 + # so that this doesn't need to depend on encoder output
702 + forward_kwargs.update(
703 + {"visual_token_mask": self.visual_token_mask})
704 + self.visual_token_mask = None
705 self.img_context_token_id = None
706
705 - if self.is_mono:
706 - forward_kwargs.update({"visual_token_mask": visual_token_mask})
707 -
707 hidden_states = self.language_model.model(**forward_kwargs)
708 return hidden_states
709
vllm/model_executor/models/molmo.py
+9 -29
@@ -13,6 +13,7 @@ from torch.nn import functional as F
13 from transformers import PretrainedConfig
14
15 from vllm.attention import Attention, AttentionMetadata
16 +from vllm.attention.layer import MultiHeadAttention
17 from vllm.compilation.decorators import support_torch_compile
18 from vllm.config import CacheConfig, VllmConfig
19 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
@@ -38,14 +39,12 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
40 from vllm.multimodal.inputs import NestedTensors
41 from vllm.multimodal.utils import cached_get_tokenizer
41 -from vllm.platforms import _Backend
42 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
43 SequenceData)
44 from vllm.transformers_utils.processor import get_processor
45
46 from .interfaces import SupportsMultiModal, SupportsPP
47 -from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
48 - is_pp_missing_parameter,
47 +from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
48 make_empty_intermediate_tensors_factory, make_layers,
49 maybe_prefix)
50
@@ -188,13 +187,11 @@ class MultiHeadDotProductAttention(nn.Module):
187 quant_config=quant_config,
188 )
189
191 - # Detect attention implementation.
192 - self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
193 - if self.attn_backend not in {
194 - _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
195 - }:
196 - raise RuntimeError(
197 - f"Molmo does not support {self.attn_backend} backend now.")
190 + self.scale = self.head_dim**-0.5
191 + self.attn = MultiHeadAttention(self.num_heads,
192 + self.head_dim,
193 + self.scale,
194 + num_kv_heads=self.num_kv_heads)
195
196 def forward(self,
197 inputs_q: torch.Tensor,
@@ -210,25 +207,8 @@ class MultiHeadDotProductAttention(nn.Module):
207 xq, _ = self.wq(inputs_q)
208 xk, _ = self.wk(inputs_k)
209 xv, _ = self.wv(inputs_v)
213 - q_shape = xq.size()[:-1] + (self.num_heads, self.head_dim)
214 - kv_shape = xk.size()[:-1] + (self.num_kv_heads, self.head_dim)
215 - xq = xq.view(*q_shape)
216 - xk = xk.view(*kv_shape)
217 - xv = xv.view(*kv_shape)
218 -
219 - if self.attn_backend == _Backend.FLASH_ATTN:
220 - from flash_attn import flash_attn_func
221 - output = flash_attn_func(xq, xk, xv, dropout_p=0.0, causal=False)
222 - elif self.attn_backend == _Backend.TORCH_SDPA:
223 - xq, xk, xv = (rearrange(x, "b s h d -> b h s d")
224 - for x in (xq, xk, xv))
225 - output = F.scaled_dot_product_attention(xq, xk, xv)
226 - output = rearrange(output, "b h s d -> b s h d ")
227 - elif self.attn_backend == _Backend.XFORMERS:
228 - from xformers import ops as xops
229 - output = xops.memory_efficient_attention_forward(xq, xk, xv, p=0)
230 -
231 - output = rearrange(output, "b s h d -> b s (h d)").contiguous()
210 +
211 + output = self.attn(xq, xk, xv)
212 output, _ = self.wo(output)
213
214 return output
vllm/model_executor/models/siglip.py
+4 -41
@@ -6,12 +6,11 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
6
7 import numpy as np
8 import torch
9 -import torch.nn.functional as F
9 from PIL import Image
10 from torch import nn
11 from transformers import SiglipVisionConfig
12
14 -from vllm.attention.selector import _Backend
13 +from vllm.attention.layer import MultiHeadAttention
14 from vllm.config import ModelConfig
15 from vllm.distributed import divide, get_tensor_model_parallel_world_size
16 from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -29,8 +28,6 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
28 resolve_visual_encoder_outputs)
29 from vllm.sequence import SequenceData
30
32 -from .utils import get_vit_attn_backend
33 -
31
32 def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
33 # Since interpolation is applied, the image size need not be divisible
@@ -291,52 +288,18 @@ class SiglipAttention(nn.Module):
288 self.tp_size = get_tensor_model_parallel_world_size()
289 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
290
294 - self.attn_backend = get_vit_attn_backend(support_fa=False)
295 - if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
296 - raise RuntimeError(
297 - f"SIGLIP does not support {self.attn_backend} backend now.")
291 + self.attn = MultiHeadAttention(self.num_heads_per_partition,
292 + self.head_dim, self.scale)
293
294 def forward(
295 self,
296 hidden_states: torch.Tensor,
297 ) -> torch.Tensor:
298 """Input shape: Batch x Time x Channel"""
304 - batch_size, q_len, _ = hidden_states.size()
305 -
299 qkv_states, _ = self.qkv_proj(hidden_states)
300 query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
301
309 - query_states = query_states.view(batch_size, q_len,
310 - self.num_heads_per_partition,
311 - self.head_dim)
312 - key_states = key_states.view(batch_size, q_len,
313 - self.num_heads_per_partition,
314 - self.head_dim)
315 - value_states = value_states.view(batch_size, q_len,
316 - self.num_heads_per_partition,
317 - self.head_dim)
318 -
319 - if self.attn_backend == _Backend.XFORMERS:
320 - from xformers import ops as xops
321 -
322 - out = xops.memory_efficient_attention_forward(query_states,
323 - key_states,
324 - value_states,
325 - p=self.dropout,
326 - scale=self.scale)
327 - elif self.attn_backend == _Backend.TORCH_SDPA:
328 - query_states, key_states, value_states = (x.transpose(1, 2)
329 - for x in (query_states,
330 - key_states,
331 - value_states))
332 - out = F.scaled_dot_product_attention(query_states,
333 - key_states,
334 - value_states,
335 - dropout_p=self.dropout,
336 - scale=self.scale)
337 - out = out.transpose(1, 2)
338 -
339 - out = out.view(batch_size, q_len, -1)
302 + out = self.attn(query_states, key_states, value_states)
303 attn_output, _ = self.out_proj(out)
304
305 return attn_output, None