@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c96fc06747
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[ROCm][AMD] Use pytorch sdpa math backend to do naive attention (#4965)

Hongxia Yang committed Jun 7, 2024 at 22:13 UTC c96fc067479453b02e92d9378eeeaebb6b3816de
1 file changed +29 -33
vllm/attention/backends/rocm_flash_attn.py
+29 -33
@@ -247,7 +247,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
247 self.use_naive_attn = True
248
249 if self.use_naive_attn:
250 - self.attn_func = _naive_attention
250 + self.attn_func = _sdpa_attention
251 logger.debug("Using naive attention in ROCmBackend")
252
253 def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
@@ -342,11 +342,18 @@ class ROCmFlashAttentionImpl(AttentionImpl):
342 # Interleave for MQA workaround.
343 key = self.repeat_kv(key, self.num_queries_per_kv)
344 value = self.repeat_kv(value, self.num_queries_per_kv)
345 + query = query.movedim(0, query.dim() - 2)
346 + key = key.movedim(0, key.dim() - 2)
347 + value = value.movedim(0, value.dim() - 2)
348 + # sdpa math backend attention
349 out = self.attn_func(
350 query,
351 key,
352 value,
353 prefill_meta.seq_lens,
354 + num_tokens,
355 + self.num_heads,
356 + self.head_size,
357 self.scale,
358 )
359 else:
@@ -402,45 +409,34 @@ class ROCmFlashAttentionImpl(AttentionImpl):
409 return output.view(num_tokens, hidden_size)
410
411
405 -def _naive_attention(
412 +def _sdpa_attention(
413 query: torch.Tensor,
414 key: torch.Tensor,
415 value: torch.Tensor,
416 seq_lens: List[int],
417 + num_tokens: int,
418 + num_heads: int,
419 + head_size: int,
420 scale: float,
421 ) -> torch.Tensor:
412 - output = torch.empty_like(query)
422 start = 0
414 - for _, seq_len in enumerate(seq_lens):
423 + output = torch.empty((num_tokens, num_heads, head_size),
424 + dtype=query.dtype,
425 + device=query.device)
426 +
427 + for seq_len in seq_lens:
428 end = start + seq_len
416 - out = _naive_masked_attention(
417 - query[start:end],
418 - key[start:end],
419 - value[start:end],
420 - scale,
421 - )
422 - # TODO(woosuk): Unnecessary copy. Optimize.
423 - output[start:end].copy_(out)
424 - start += seq_len
429 + with torch.backends.cuda.sdp_kernel(enable_math=True,
430 + enable_flash=False,
431 + enable_mem_efficient=False):
432 + sub_out = torch.nn.functional.scaled_dot_product_attention(
433 + query[:, start:end, :],
434 + key[:, start:end, :],
435 + value[:, start:end, :],
436 + dropout_p=0.0,
437 + is_causal=True,
438 + scale=scale).movedim(query.dim() - 2, 0)
439 + output[start:end, :, :] = sub_out
440 + start = end
441
442 return output
427 -
428 -
429 -def _naive_masked_attention(
430 - query: torch.Tensor,
431 - key: torch.Tensor,
432 - value: torch.Tensor,
433 - scale: float,
434 -) -> torch.Tensor:
435 - seq_len, head_size, head_dim = query.shape
436 - attn_mask = torch.triu(torch.ones(seq_len,
437 - seq_len,
438 - dtype=query.dtype,
439 - device=query.device),
440 - diagonal=1)
441 - attn_mask = attn_mask * torch.finfo(query.dtype).min
442 - attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
443 - attn_weights = attn_weights + attn_mask.float()
444 - attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
445 - out = torch.einsum("hqk,khd->qhd", attn_weights, value)
446 - return out