A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[ROCm][AMD] Use pytorch sdpa math backend to do naive attention (#4965)
Hongxia Yang committed
Jun 7, 2024 at 22:13 UTC
c96fc067479453b02e92d9378eeeaebb6b3816de
1 file changed
+29
-33
vllm/attention/backends/rocm_flash_attn.py
+29
-33
@@ -247,7 +247,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
247
self.use_naive_attn = True
248
249
if self.use_naive_attn:
250
- self.attn_func = _naive_attention
250
+ self.attn_func = _sdpa_attention
251
logger.debug("Using naive attention in ROCmBackend")
252
253
def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
@@ -342,11 +342,18 @@ class ROCmFlashAttentionImpl(AttentionImpl):
342
# Interleave for MQA workaround.
343
key = self.repeat_kv(key, self.num_queries_per_kv)
344
value = self.repeat_kv(value, self.num_queries_per_kv)
345
+ query = query.movedim(0, query.dim() - 2)
346
+ key = key.movedim(0, key.dim() - 2)
347
+ value = value.movedim(0, value.dim() - 2)
348
+ # sdpa math backend attention
349
out = self.attn_func(
350
query,
351
key,
352
value,
353
prefill_meta.seq_lens,
354
+ num_tokens,
355
+ self.num_heads,
356
+ self.head_size,
357
self.scale,
358
)
359
else:
@@ -402,45 +409,34 @@ class ROCmFlashAttentionImpl(AttentionImpl):
409
return output.view(num_tokens, hidden_size)
410
411
405
-def _naive_attention(
412
+def _sdpa_attention(
413
query: torch.Tensor,
414
key: torch.Tensor,
415
value: torch.Tensor,
416
seq_lens: List[int],
417
+ num_tokens: int,
418
+ num_heads: int,
419
+ head_size: int,
420
scale: float,
421
) -> torch.Tensor:
412
- output = torch.empty_like(query)
422
start = 0
414
- for _, seq_len in enumerate(seq_lens):
423
+ output = torch.empty((num_tokens, num_heads, head_size),
424
+ dtype=query.dtype,
425
+ device=query.device)
426
+
427
+ for seq_len in seq_lens:
428
end = start + seq_len
416
- out = _naive_masked_attention(
417
- query[start:end],
418
- key[start:end],
419
- value[start:end],
420
- scale,
421
- )
422
- # TODO(woosuk): Unnecessary copy. Optimize.
423
- output[start:end].copy_(out)
424
- start += seq_len
429
+ with torch.backends.cuda.sdp_kernel(enable_math=True,
430
+ enable_flash=False,
431
+ enable_mem_efficient=False):
432
+ sub_out = torch.nn.functional.scaled_dot_product_attention(
433
+ query[:, start:end, :],
434
+ key[:, start:end, :],
435
+ value[:, start:end, :],
436
+ dropout_p=0.0,
437
+ is_causal=True,
438
+ scale=scale).movedim(query.dim() - 2, 0)
439
+ output[start:end, :, :] = sub_out
440
+ start = end
441
442
return output
427
-
428
-
429
-def _naive_masked_attention(
430
- query: torch.Tensor,
431
- key: torch.Tensor,
432
- value: torch.Tensor,
433
- scale: float,
434
-) -> torch.Tensor:
435
- seq_len, head_size, head_dim = query.shape
436
- attn_mask = torch.triu(torch.ones(seq_len,
437
- seq_len,
438
- dtype=query.dtype,
439
- device=query.device),
440
- diagonal=1)
441
- attn_mask = attn_mask * torch.finfo(query.dtype).min
442
- attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
443
- attn_weights = attn_weights + attn_mask.float()
444
- attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
445
- out = torch.einsum("hqk,khd->qhd", attn_weights, value)
446
- return out