A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Misc] Revert FA on ViT #12355 and #12435 (#12445)
Roger Wang committed
Jan 26, 2025 at 03:56 UTC
a5255270c3ad492b5def19fe38beb9b2df30e74f
1 file changed
+4
-37
vllm/attention/layer.py
+4
-37
@@ -210,9 +210,6 @@ class MultiHeadAttention(nn.Module):
210
self.scale = scale
211
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
212
213
- assert self.num_heads % self.num_kv_heads == 0
214
- self.num_queries_per_kv = self.num_heads // self.num_kv_heads
215
-
213
dtype = torch.get_default_dtype()
214
attn_backend = get_attn_backend(head_size,
215
dtype,
@@ -220,12 +217,12 @@ class MultiHeadAttention(nn.Module):
217
block_size=16,
218
is_attention_free=False)
219
backend = backend_name_to_enum(attn_backend.get_name())
220
+ if backend in {_Backend.FLASH_ATTN, _Backend.FLASH_ATTN_VLLM_V1}:
221
+ backend = _Backend.XFORMERS
222
223
self.attn_backend = backend if backend in {
224
_Backend.TORCH_SDPA,
225
_Backend.XFORMERS,
227
- _Backend.FLASH_ATTN,
228
- _Backend.FLASH_ATTN_VLLM_V1,
226
} else _Backend.TORCH_SDPA
227
228
def forward(
@@ -235,6 +232,7 @@ class MultiHeadAttention(nn.Module):
232
value: torch.Tensor,
233
) -> torch.Tensor:
234
"""Input shape: batch_size x seq_len x hidden_size"""
235
+ # TODO(Isotr0py): Use existing backend implementations and support FA3
236
bsz, q_len, _ = query.size()
237
kv_len = key.size(1)
238
@@ -242,38 +240,7 @@ class MultiHeadAttention(nn.Module):
240
key = key.view(bsz, kv_len, self.num_kv_heads, self.head_size)
241
value = value.view(bsz, kv_len, self.num_kv_heads, self.head_size)
242
245
- if (num_repeat := self.num_queries_per_kv) > 1:
246
- # Handle MQA and GQA
247
- key = torch.repeat_interleave(key, num_repeat, dim=2)
248
- value = torch.repeat_interleave(value, num_repeat, dim=2)
249
-
250
- if self.attn_backend in {
251
- _Backend.FLASH_ATTN,
252
- _Backend.FLASH_ATTN_VLLM_V1,
253
- }:
254
- from vllm.vllm_flash_attn import flash_attn_varlen_func
255
-
256
- cu_seqlens_q = torch.arange(0, (bsz + 1) * q_len,
257
- step=q_len,
258
- dtype=torch.int32,
259
- device=query.device)
260
- cu_seqlens_k = torch.arange(0, (bsz + 1) * kv_len,
261
- step=kv_len,
262
- dtype=torch.int32,
263
- device=key.device)
264
-
265
- out = flash_attn_varlen_func(
266
- query.flatten(0, 1),
267
- key.flatten(0, 1),
268
- value.flatten(0, 1),
269
- cu_seqlens_q=cu_seqlens_q,
270
- cu_seqlens_k=cu_seqlens_k,
271
- max_seqlen_q=q_len,
272
- max_seqlen_k=kv_len,
273
- softmax_scale=self.scale,
274
- )
275
- out = out.reshape(bsz, q_len, -1)
276
- elif self.attn_backend == _Backend.XFORMERS:
243
+ if self.attn_backend == _Backend.XFORMERS:
244
from xformers import ops as xops
245
246
out = xops.memory_efficient_attention_forward(query,