A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Consolidate ViTs attention implementation without mask (#10893)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Dec 5, 2024 at 02:11 UTC
10398b4706ee71d0bddc32c1d33b11e73df12a27
9 files changed
+109
-226
vllm/attention/layer.py
+63
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional
3
4
import torch
5
import torch.nn as nn
6
+import torch.nn.functional as F
7
8
from vllm.attention import AttentionMetadata, AttentionType
9
from vllm.attention.selector import backend_name_to_enum, get_attn_backend
@@ -168,6 +169,68 @@ class Attention(nn.Module):
169
return s
170
171
172
+class MultiHeadAttention(nn.Module):
173
+ """Multi-headed attention without any cache, used for ViT."""
174
+
175
+ def __init__(
176
+ self,
177
+ num_heads: int,
178
+ head_size: int,
179
+ scale: float,
180
+ num_kv_heads: Optional[int] = None,
181
+ ):
182
+ super().__init__()
183
+ self.num_heads = num_heads
184
+ self.head_size = head_size
185
+ self.scale = scale
186
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
187
+
188
+ dtype = torch.get_default_dtype()
189
+ attn_backend = get_attn_backend(head_size,
190
+ dtype,
191
+ kv_cache_dtype=None,
192
+ block_size=16,
193
+ is_attention_free=False)
194
+ if attn_backend in {_Backend.FLASH_ATTN, _Backend.FLASH_ATTN_VLLM_V1}:
195
+ attn_backend = _Backend.XFORMERS
196
+
197
+ self.attn_backend = attn_backend if attn_backend in {
198
+ _Backend.TORCH_SDPA, _Backend.XFORMERS
199
+ } else _Backend.TORCH_SDPA
200
+
201
+ def forward(
202
+ self,
203
+ query: torch.Tensor,
204
+ key: torch.Tensor,
205
+ value: torch.Tensor,
206
+ ) -> torch.Tensor:
207
+ """Input shape: batch_size x seq_len x hidden_size"""
208
+ # TODO(Isotr0py): Use existing backend implementations and support FA2
209
+ bsz, q_len, _ = query.size()
210
+ kv_len = key.size(1)
211
+
212
+ query = query.view(bsz, q_len, self.num_heads, self.head_size)
213
+ key = key.view(bsz, kv_len, self.num_kv_heads, self.head_size)
214
+ value = value.view(bsz, kv_len, self.num_kv_heads, self.head_size)
215
+
216
+ if self.attn_backend == _Backend.XFORMERS:
217
+ from xformers import ops as xops
218
+
219
+ out = xops.memory_efficient_attention_forward(query,
220
+ key,
221
+ value,
222
+ scale=self.scale)
223
+ elif self.attn_backend == _Backend.TORCH_SDPA:
224
+ query, key, value = (x.transpose(1, 2)
225
+ for x in (query, key, value))
226
+ out = F.scaled_dot_product_attention(query,
227
+ key,
228
+ value,
229
+ scale=self.scale)
230
+ out = out.transpose(1, 2)
231
+ return out.view(bsz, q_len, -1)
232
+
233
+
234
def unified_attention(
235
query: torch.Tensor,
236
key: torch.Tensor,
vllm/model_executor/models/blip.py
+4
-41
@@ -4,11 +4,10 @@ from typing import Iterable, Optional, Set, Tuple, Union
4
5
import torch
6
import torch.nn as nn
7
-import torch.nn.functional as F
7
from PIL import Image
8
from transformers import Blip2VisionConfig, BlipVisionConfig
9
11
-from vllm.attention.selector import _Backend
10
+from vllm.attention.layer import MultiHeadAttention
11
from vllm.config import ModelConfig
12
from vllm.distributed import divide, get_tensor_model_parallel_world_size
13
from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -22,8 +21,6 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
21
repeat_and_pad_placeholder_tokens)
22
from vllm.sequence import SequenceData
23
25
-from .utils import get_vit_attn_backend
26
-
24
25
def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
26
assert image_size % patch_size == 0
@@ -205,11 +202,8 @@ class BlipAttention(nn.Module):
202
self.tp_size = get_tensor_model_parallel_world_size()
203
self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
204
208
- # Detect attention implementation.
209
- self.attn_backend = get_vit_attn_backend(support_fa=False)
210
- if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
211
- raise RuntimeError(
212
- f"BLIP does not support {self.attn_backend} backend now.")
205
+ self.attn = MultiHeadAttention(self.num_heads_per_partition,
206
+ self.head_dim, self.scale)
207
208
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
209
return tensor.view(bsz, seq_len, self.num_heads,
@@ -220,41 +214,10 @@ class BlipAttention(nn.Module):
214
hidden_states: torch.Tensor,
215
):
216
"""Input shape: Batch x Time x Channel"""
223
- bsz, tgt_len, _ = hidden_states.size()
217
218
qkv_states, _ = self.qkv(hidden_states)
219
query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
227
- query_states = query_states.view(bsz, tgt_len,
228
- self.num_heads_per_partition,
229
- self.head_dim)
230
- key_states = key_states.view(bsz, tgt_len,
231
- self.num_heads_per_partition,
232
- self.head_dim)
233
- value_states = value_states.view(bsz, tgt_len,
234
- self.num_heads_per_partition,
235
- self.head_dim)
236
-
237
- if self.attn_backend == _Backend.XFORMERS:
238
- from xformers import ops as xops
239
-
240
- out = xops.memory_efficient_attention_forward(query_states,
241
- key_states,
242
- value_states,
243
- p=self.dropout,
244
- scale=self.scale)
245
- elif self.attn_backend == _Backend.TORCH_SDPA:
246
- query_states, key_states, value_states = (x.transpose(1, 2)
247
- for x in (query_states,
248
- key_states,
249
- value_states))
250
- out = F.scaled_dot_product_attention(query_states,
251
- key_states,
252
- value_states,
253
- dropout_p=self.dropout,
254
- scale=self.scale)
255
- out = out.transpose(1, 2)
256
-
257
- out = out.view(bsz, tgt_len, -1)
220
+ out = self.attn(query_states, key_states, value_states)
221
attn_output, _ = self.projection(out)
222
223
return attn_output, None
vllm/model_executor/models/clip.py
+4
-42
@@ -5,11 +5,10 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
5
import numpy as np
6
import torch
7
import torch.nn as nn
8
-import torch.nn.functional as F
8
from PIL import Image
9
from transformers import CLIPVisionConfig
10
12
-from vllm.attention.selector import _Backend
11
+from vllm.attention.layer import MultiHeadAttention
12
from vllm.config import ModelConfig
13
from vllm.distributed import divide, get_tensor_model_parallel_world_size
14
from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -25,8 +24,6 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
24
resolve_visual_encoder_outputs)
25
from vllm.sequence import SequenceData
26
28
-from .utils import get_vit_attn_backend
29
-
27
28
def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
29
assert image_size % patch_size == 0
@@ -235,11 +232,8 @@ class CLIPAttention(nn.Module):
232
self.tp_size = get_tensor_model_parallel_world_size()
233
self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
234
238
- # Detect attention implementation.
239
- self.attn_backend = get_vit_attn_backend(support_fa=False)
240
- if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
241
- raise RuntimeError(
242
- f"CLIP does not support {self.attn_backend} backend now.")
235
+ self.attn = MultiHeadAttention(self.num_heads_per_partition,
236
+ self.head_dim, self.scale)
237
238
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
239
return tensor.view(bsz, seq_len, self.num_heads,
@@ -250,42 +244,10 @@ class CLIPAttention(nn.Module):
244
hidden_states: torch.Tensor,
245
):
246
"""Input shape: Batch x Time x Channel"""
253
- bsz, tgt_len, _ = hidden_states.size()
247
248
qkv_states, _ = self.qkv_proj(hidden_states)
249
query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
257
-
258
- query_states = query_states.view(bsz, tgt_len,
259
- self.num_heads_per_partition,
260
- self.head_dim)
261
- key_states = key_states.view(bsz, tgt_len,
262
- self.num_heads_per_partition,
263
- self.head_dim)
264
- value_states = value_states.view(bsz, tgt_len,
265
- self.num_heads_per_partition,
266
- self.head_dim)
267
-
268
- if self.attn_backend == _Backend.XFORMERS:
269
- from xformers import ops as xops
270
-
271
- out = xops.memory_efficient_attention_forward(query_states,
272
- key_states,
273
- value_states,
274
- p=self.dropout,
275
- scale=self.scale)
276
- elif self.attn_backend == _Backend.TORCH_SDPA:
277
- query_states, key_states, value_states = (x.transpose(1, 2)
278
- for x in (query_states,
279
- key_states,
280
- value_states))
281
- out = F.scaled_dot_product_attention(query_states,
282
- key_states,
283
- value_states,
284
- dropout_p=self.dropout,
285
- scale=self.scale)
286
- out = out.transpose(1, 2)
287
-
288
- out = out.view(bsz, tgt_len, -1)
250
+ out = self.attn(query_states, key_states, value_states)
251
attn_output, _ = self.out_proj(out)
252
253
return attn_output, None
vllm/model_executor/models/glm4_vision_encoder.py
+6
-16
@@ -8,6 +8,7 @@ import torch
8
from torch import nn
9
from torch.nn import LayerNorm
10
11
+from vllm.attention.layer import MultiHeadAttention
12
from vllm.distributed import get_tensor_model_parallel_world_size
13
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
14
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -77,27 +78,16 @@ class Attention(nn.Module):
78
quant_config=quant_config,
79
)
80
81
+ self.attn = MultiHeadAttention(self.num_heads_per_rank, self.head_dim,
82
+ self.scale)
83
self.output_dropout = torch.nn.Dropout(config.dropout_prob)
84
85
def forward(self, x: torch.Tensor) -> torch.Tensor:
83
- B, L, _ = x.shape
86
qkv, _ = self.query_key_value(x) # B, L, 3 * H * D
87
q, k, v = qkv.chunk(3, dim=-1)
86
- q = q.reshape(B, L, self.num_heads_per_rank,
87
- self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
88
- k = k.reshape(B, L, self.num_heads_per_rank,
89
- self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
90
- v = v.reshape(B, L, self.num_heads_per_rank,
91
- self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
92
-
93
- out = torch.nn.functional.scaled_dot_product_attention(q,
94
- k,
95
- v,
96
- attn_mask=None,
97
- dropout_p=0.,
98
- is_causal=False)
99
-
100
- output, _ = self.dense(out.transpose(1, 2).view(B, L, -1))
88
+
89
+ out = self.attn(q, k, v)
90
+ output, _ = self.dense(out)
91
output = self.output_dropout(output)
92
return output
93
vllm/model_executor/models/idefics2_vision_model.py
+4
-21
@@ -21,8 +21,8 @@ import torch
21
from torch import nn
22
from transformers.models.idefics2.configuration_idefics2 import (
23
Idefics2Config, Idefics2VisionConfig)
24
-from xformers import ops as xops
24
25
+from vllm.attention.layer import MultiHeadAttention
26
from vllm.distributed import divide, get_tensor_model_parallel_world_size
27
from vllm.model_executor.layers.activation import get_act_fn
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -141,35 +141,18 @@ class Idefics2VisionAttention(nn.Module):
141
)
142
self.tp_size = get_tensor_model_parallel_world_size()
143
self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
144
- self.is_causal = False
144
+ self.attn = MultiHeadAttention(self.num_heads_per_partition,
145
+ self.head_dim, self.scale)
146
147
def forward(
148
self,
149
hidden_states: torch.Tensor,
150
) -> torch.Tensor:
150
- batch_size, q_len, _ = hidden_states.size()
151
qkv, _ = self.qkv_proj(
152
hidden_states
153
) # batch_size, q_len, 3 * num_heads_per_partition * head_dim
154
query_states, key_states, value_states = qkv.chunk(3, dim=-1)
155
- query_states = query_states.view(batch_size, q_len,
156
- self.num_heads_per_partition,
157
- self.head_dim)
158
- key_states = key_states.view(batch_size, q_len,
159
- self.num_heads_per_partition,
160
- self.head_dim)
161
- value_states = value_states.view(batch_size, q_len,
162
- self.num_heads_per_partition,
163
- self.head_dim)
164
- # see: https://facebookresearch.github.io/xformers/components/ops.html
165
- out = xops.memory_efficient_attention_forward(
166
- query_states,
167
- key_states,
168
- value_states,
169
- p=self.dropout,
170
- scale=self.scale,
171
- )
172
- out = out.view(batch_size, q_len, -1)
155
+ out = self.attn(query_states, key_states, value_states)
156
attn_output, _ = self.out_proj(out)
157
return attn_output
158
vllm/model_executor/models/intern_vit.py
+4
-24
@@ -12,7 +12,7 @@ import torch.nn as nn
12
import torch.nn.functional as F
13
from transformers import PretrainedConfig
14
15
-from vllm.attention.selector import _Backend
15
+from vllm.attention.layer import MultiHeadAttention
16
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
17
get_tensor_model_parallel_world_size,
18
split_tensor_along_last_dim,
@@ -25,8 +25,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
25
from vllm.model_executor.layers.quantization import QuantizationConfig
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
28
-from .utils import get_vit_attn_backend
29
-
28
NORM2FN = {
29
'rms_norm': RMSNorm,
30
'layer_norm': nn.LayerNorm,
@@ -183,10 +181,8 @@ class InternParallelAttention(nn.Module):
181
prefix=f"{prefix}.proj",
182
)
183
186
- self.attn_backend = get_vit_attn_backend(support_fa=False)
187
- if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
188
- raise RuntimeError(
189
- f"InternViT does not support {self.attn_backend} backend now.")
184
+ self.attn = MultiHeadAttention(self.num_heads_per_partition,
185
+ self.head_dim, self.scale)
186
187
def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
188
if self.tp_size > 1:
@@ -209,23 +205,7 @@ class InternParallelAttention(nn.Module):
205
if self.qk_normalization:
206
q, k = self._apply_qk_norm(q, k)
207
212
- q = q.view(B, N, self.num_heads_per_partition, self.head_dim)
213
- k = k.view(B, N, self.num_heads_per_partition, self.head_dim)
214
- v = v.view(B, N, self.num_heads_per_partition, self.head_dim)
215
-
216
- if self.attn_backend == _Backend.XFORMERS:
217
- from xformers import ops as xops
218
-
219
- out = xops.memory_efficient_attention_forward(q,
220
- k,
221
- v,
222
- scale=self.scale)
223
- elif self.attn_backend == _Backend.TORCH_SDPA:
224
- q, k, v = (x.transpose(1, 2) for x in (q, k, v))
225
- out = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
226
- out = out.transpose(1, 2)
227
-
228
- out = out.view(B, N, -1)
208
+ out = self.attn(q, k, v)
209
out, _ = self.proj(out)
210
return out
211
vllm/model_executor/models/internvl.py
+11
-12
@@ -482,6 +482,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
482
self.mlp1 = self._init_mlp1(config)
483
484
self.img_context_token_id = None
485
+ self.visual_token_mask = None
486
self.make_empty_intermediate_tensors = (
487
self.language_model.make_empty_intermediate_tensors)
488
@@ -635,13 +636,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
636
637
return image_embeds
638
638
- def _get_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
639
+ def _set_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
640
if self.is_mono:
640
- visual_token_mask = (
641
+ self.visual_token_mask = (
642
input_ids == self.img_context_token_id).reshape(-1, 1)
643
else:
643
- visual_token_mask = None
644
- return visual_token_mask
644
+ self.visual_token_mask = None
645
646
def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
647
image_input = self._parse_and_validate_image_input(**kwargs)
@@ -658,6 +658,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
658
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
659
if multimodal_embeddings is not None:
660
assert self.img_context_token_id is not None
661
+ self._set_visual_token_mask(input_ids)
662
inputs_embeds = merge_multimodal_embeddings(
663
input_ids, inputs_embeds, multimodal_embeddings,
664
self.img_context_token_id)
@@ -674,7 +675,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
675
**kwargs: object,
676
) -> Union[SamplerOutput, IntermediateTensors]:
677
677
- visual_token_mask = None
678
if intermediate_tensors is not None:
679
input_ids = None
680
inputs_embeds = None
@@ -695,16 +695,15 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
695
"intermediate_tensors": intermediate_tensors,
696
"inputs_embeds": inputs_embeds,
697
}
698
- if self.img_context_token_id is not None:
699
- visual_token_mask = self._get_visual_token_mask(input_ids)
698
701
- # We always overwrite it back to None after computing visual token
702
- # mask so that this doesn't need to depend on encoder output
699
+ if self.visual_token_mask is not None:
700
+ # overwrite visual_token_mask and img_context_token_id back to None,
701
+ # so that this doesn't need to depend on encoder output
702
+ forward_kwargs.update(
703
+ {"visual_token_mask": self.visual_token_mask})
704
+ self.visual_token_mask = None
705
self.img_context_token_id = None
706
705
- if self.is_mono:
706
- forward_kwargs.update({"visual_token_mask": visual_token_mask})
707
-
707
hidden_states = self.language_model.model(**forward_kwargs)
708
return hidden_states
709
vllm/model_executor/models/molmo.py
+9
-29
@@ -13,6 +13,7 @@ from torch.nn import functional as F
13
from transformers import PretrainedConfig
14
15
from vllm.attention import Attention, AttentionMetadata
16
+from vllm.attention.layer import MultiHeadAttention
17
from vllm.compilation.decorators import support_torch_compile
18
from vllm.config import CacheConfig, VllmConfig
19
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
@@ -38,14 +39,12 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
40
from vllm.multimodal.inputs import NestedTensors
41
from vllm.multimodal.utils import cached_get_tokenizer
41
-from vllm.platforms import _Backend
42
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
43
SequenceData)
44
from vllm.transformers_utils.processor import get_processor
45
46
from .interfaces import SupportsMultiModal, SupportsPP
47
-from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
48
- is_pp_missing_parameter,
47
+from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
48
make_empty_intermediate_tensors_factory, make_layers,
49
maybe_prefix)
50
@@ -188,13 +187,11 @@ class MultiHeadDotProductAttention(nn.Module):
187
quant_config=quant_config,
188
)
189
191
- # Detect attention implementation.
192
- self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
193
- if self.attn_backend not in {
194
- _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
195
- }:
196
- raise RuntimeError(
197
- f"Molmo does not support {self.attn_backend} backend now.")
190
+ self.scale = self.head_dim**-0.5
191
+ self.attn = MultiHeadAttention(self.num_heads,
192
+ self.head_dim,
193
+ self.scale,
194
+ num_kv_heads=self.num_kv_heads)
195
196
def forward(self,
197
inputs_q: torch.Tensor,
@@ -210,25 +207,8 @@ class MultiHeadDotProductAttention(nn.Module):
207
xq, _ = self.wq(inputs_q)
208
xk, _ = self.wk(inputs_k)
209
xv, _ = self.wv(inputs_v)
213
- q_shape = xq.size()[:-1] + (self.num_heads, self.head_dim)
214
- kv_shape = xk.size()[:-1] + (self.num_kv_heads, self.head_dim)
215
- xq = xq.view(*q_shape)
216
- xk = xk.view(*kv_shape)
217
- xv = xv.view(*kv_shape)
218
-
219
- if self.attn_backend == _Backend.FLASH_ATTN:
220
- from flash_attn import flash_attn_func
221
- output = flash_attn_func(xq, xk, xv, dropout_p=0.0, causal=False)
222
- elif self.attn_backend == _Backend.TORCH_SDPA:
223
- xq, xk, xv = (rearrange(x, "b s h d -> b h s d")
224
- for x in (xq, xk, xv))
225
- output = F.scaled_dot_product_attention(xq, xk, xv)
226
- output = rearrange(output, "b h s d -> b s h d ")
227
- elif self.attn_backend == _Backend.XFORMERS:
228
- from xformers import ops as xops
229
- output = xops.memory_efficient_attention_forward(xq, xk, xv, p=0)
230
-
231
- output = rearrange(output, "b s h d -> b s (h d)").contiguous()
210
+
211
+ output = self.attn(xq, xk, xv)
212
output, _ = self.wo(output)
213
214
return output
vllm/model_executor/models/siglip.py
+4
-41
@@ -6,12 +6,11 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
6
7
import numpy as np
8
import torch
9
-import torch.nn.functional as F
9
from PIL import Image
10
from torch import nn
11
from transformers import SiglipVisionConfig
12
14
-from vllm.attention.selector import _Backend
13
+from vllm.attention.layer import MultiHeadAttention
14
from vllm.config import ModelConfig
15
from vllm.distributed import divide, get_tensor_model_parallel_world_size
16
from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -29,8 +28,6 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
28
resolve_visual_encoder_outputs)
29
from vllm.sequence import SequenceData
30
32
-from .utils import get_vit_attn_backend
33
-
31
32
def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
33
# Since interpolation is applied, the image size need not be divisible
@@ -291,52 +288,18 @@ class SiglipAttention(nn.Module):
288
self.tp_size = get_tensor_model_parallel_world_size()
289
self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
290
294
- self.attn_backend = get_vit_attn_backend(support_fa=False)
295
- if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
296
- raise RuntimeError(
297
- f"SIGLIP does not support {self.attn_backend} backend now.")
291
+ self.attn = MultiHeadAttention(self.num_heads_per_partition,
292
+ self.head_dim, self.scale)
293
294
def forward(
295
self,
296
hidden_states: torch.Tensor,
297
) -> torch.Tensor:
298
"""Input shape: Batch x Time x Channel"""
304
- batch_size, q_len, _ = hidden_states.size()
305
-
299
qkv_states, _ = self.qkv_proj(hidden_states)
300
query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
301
309
- query_states = query_states.view(batch_size, q_len,
310
- self.num_heads_per_partition,
311
- self.head_dim)
312
- key_states = key_states.view(batch_size, q_len,
313
- self.num_heads_per_partition,
314
- self.head_dim)
315
- value_states = value_states.view(batch_size, q_len,
316
- self.num_heads_per_partition,
317
- self.head_dim)
318
-
319
- if self.attn_backend == _Backend.XFORMERS:
320
- from xformers import ops as xops
321
-
322
- out = xops.memory_efficient_attention_forward(query_states,
323
- key_states,
324
- value_states,
325
- p=self.dropout,
326
- scale=self.scale)
327
- elif self.attn_backend == _Backend.TORCH_SDPA:
328
- query_states, key_states, value_states = (x.transpose(1, 2)
329
- for x in (query_states,
330
- key_states,
331
- value_states))
332
- out = F.scaled_dot_product_attention(query_states,
333
- key_states,
334
- value_states,
335
- dropout_p=self.dropout,
336
- scale=self.scale)
337
- out = out.transpose(1, 2)
338
-
339
- out = out.view(batch_size, q_len, -1)
302
+ out = self.attn(query_states, key_states, value_states)
303
attn_output, _ = self.out_proj(out)
304
305
return attn_output, None