@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 56b325e977
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ROCm][AMD][Model]Adding alibi slopes support in ROCm triton flash attention and naive flash attention (#6043)

Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>

Gregory Shtrasberg committed Jul 4, 2024 at 01:19 UTC 56b325e977435af744f8b3dca7af0ca209663558
1 file changed +51 -2
vllm/attention/backends/rocm_flash_attn.py
+51 -2
@@ -166,6 +166,37 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
166 return self._cached_decode_metadata
167
168
169 +def _make_alibi_bias(alibi_slopes: torch.Tensor,
170 + dtype: torch.dtype,
171 + seq_lens: Optional[List[int]],
172 + make_attn_mask: bool = True) -> List[torch.Tensor]:
173 + attn_biases = []
174 + if seq_lens:
175 + for seq_len in seq_lens:
176 + bias = torch.arange(seq_len, dtype=dtype)
177 + # NOTE(zhuohan): HF uses
178 + # `bias = bias[None, :].repeat(seq_len, 1)`
179 + # here. We find that both biases give the same results, but
180 + # the bias below more accurately follows the original ALiBi
181 + # paper.
182 + bias = bias[None, :] - bias[:, None]
183 +
184 + num_heads = alibi_slopes.shape[0]
185 + bias = bias[None, :].repeat(
186 + (num_heads, 1, 1)).to(alibi_slopes.device)
187 + bias.mul_(alibi_slopes[:, None, None])
188 + if make_attn_mask:
189 + inf_mask = torch.empty(
190 + (1, seq_len, seq_len),
191 + dtype=bias.dtype).fill_(-torch.inf).triu_(diagonal=1).to(
192 + alibi_slopes.device)
193 + attn_biases.append((bias + inf_mask).to(dtype))
194 + else:
195 + attn_biases.append(bias.to(dtype))
196 +
197 + return attn_biases
198 +
199 +
200 class ROCmFlashAttentionImpl(AttentionImpl):
201 """
202 If the input tensors contain prompt tokens, the layout is as follows:
@@ -324,7 +355,14 @@ class ROCmFlashAttentionImpl(AttentionImpl):
355 # triton attention
356 # When block_tables are not filled, it means q and k are the
357 # prompt, and they have the same length.
358 + attn_masks = None
359 if self.use_triton_flash_attn:
360 + if self.alibi_slopes is not None:
361 + attn_masks = _make_alibi_bias(
362 + self.alibi_slopes,
363 + query.dtype,
364 + attn_metadata.seq_lens,
365 + make_attn_mask=False) # type: ignore
366 out, _ = self.attn_func(
367 query,
368 key,
@@ -336,12 +374,20 @@ class ROCmFlashAttentionImpl(AttentionImpl):
374 prefill_meta.max_prefill_seq_len,
375 True,
376 self.scale,
377 + attn_masks[0][None]
378 + if attn_masks is not None else None,
379 )
380 elif self.use_naive_attn:
381 if self.num_kv_heads != self.num_heads:
382 # Interleave for MQA workaround.
383 key = self.repeat_kv(key, self.num_queries_per_kv)
384 value = self.repeat_kv(value, self.num_queries_per_kv)
385 + if self.alibi_slopes is not None:
386 + attn_masks = _make_alibi_bias(
387 + self.alibi_slopes,
388 + query.dtype,
389 + attn_metadata.seq_lens,
390 + make_attn_mask=True) # type: ignore
391 query = query.movedim(0, query.dim() - 2)
392 key = key.movedim(0, key.dim() - 2)
393 value = value.movedim(0, value.dim() - 2)
@@ -355,6 +401,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
401 self.num_heads,
402 self.head_size,
403 self.scale,
404 + attn_masks,
405 )
406 else:
407 out = self.attn_func(
@@ -418,13 +465,14 @@ def _sdpa_attention(
465 num_heads: int,
466 head_size: int,
467 scale: float,
468 + attn_masks: Optional[List[torch.Tensor]] = None,
469 ) -> torch.Tensor:
470 start = 0
471 output = torch.empty((num_tokens, num_heads, head_size),
472 dtype=query.dtype,
473 device=query.device)
474
427 - for seq_len in seq_lens:
475 + for i, seq_len in enumerate(seq_lens):
476 end = start + seq_len
477 with torch.backends.cuda.sdp_kernel(enable_math=True,
478 enable_flash=False,
@@ -434,7 +482,8 @@ def _sdpa_attention(
482 key[:, start:end, :],
483 value[:, start:end, :],
484 dropout_p=0.0,
437 - is_causal=True,
485 + is_causal=attn_masks is None,
486 + attn_mask=attn_masks[i] if attn_masks else None,
487 scale=scale).movedim(query.dim() - 2, 0)
488 output[start:end, :, :] = sub_out
489 start = end