@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18d23f642a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[ROCm][Hardware][AMD] Enable group query attention for triton FA (#4406)

Hongxia Yang committed Apr 27, 2024 at 02:37 UTC 18d23f642af9c56f45ccf684a22f386fc54c6ead
2 files changed +36 -41
vllm/attention/backends/rocm_flash_attn.py
+25 -28
@@ -253,36 +253,31 @@ class ROCmFlashAttentionImpl(AttentionImpl):
253 # triton attention
254 # When block_tables are not filled, it means q and k are the
255 # prompt, and they have the same length.
256 - if self.use_triton_flash_attn or self.use_naive_attn:
256 + if self.use_triton_flash_attn:
257 + out, _ = self.attn_func(
258 + query,
259 + key,
260 + value,
261 + None,
262 + prefill_meta.seq_start_loc,
263 + prefill_meta.seq_start_loc,
264 + prefill_meta.max_prompt_len,
265 + prefill_meta.max_prompt_len,
266 + True,
267 + self.scale,
268 + )
269 + elif self.use_naive_attn:
270 if self.num_kv_heads != self.num_heads:
271 # Interleave for MQA workaround.
272 key = self.repeat_kv(key, self.num_queries_per_kv)
273 value = self.repeat_kv(value, self.num_queries_per_kv)
261 - if self.use_naive_attn:
262 - out = self.attn_func(
263 - query,
264 - key,
265 - value,
266 - prefill_meta.prompt_lens,
267 - self.scale,
268 - )
269 - assert output[:num_prefill_tokens].shape == out.shape
270 - output[:num_prefill_tokens] = out
271 - else:
272 - out, _ = self.attn_func(
273 - query,
274 - key,
275 - value,
276 - None,
277 - prefill_meta.seq_start_loc,
278 - prefill_meta.seq_start_loc,
279 - prefill_meta.max_prompt_len,
280 - prefill_meta.max_prompt_len,
281 - True,
282 - self.scale,
283 - )
284 - assert output[:num_prefill_tokens].shape == out.shape
285 - output[:num_prefill_tokens] = out
274 + out = self.attn_func(
275 + query,
276 + key,
277 + value,
278 + prefill_meta.prompt_lens,
279 + self.scale,
280 + )
281 else:
282 out = self.attn_func(
283 q=query,
@@ -295,8 +290,10 @@ class ROCmFlashAttentionImpl(AttentionImpl):
290 softmax_scale=self.scale,
291 causal=True,
292 )
298 - assert output[:num_prefill_tokens].shape == out.shape
299 - output[:num_prefill_tokens] = out
293 +
294 + # common code for prefill
295 + assert output[:num_prefill_tokens].shape == out.shape
296 + output[:num_prefill_tokens] = out
297 else:
298 # prefix-enabled attention
299 output[:num_prefill_tokens] = PagedAttention.forward_prefix(
vllm/attention/ops/triton_flash_attention.py
+11 -13
@@ -293,7 +293,7 @@ def _attn_fwd_inner(
293 num_warps=4,
294 ),
295 ],
296 - key=["hq", "hk", "IS_CAUSAL", "dropout_p", "BLOCK_DMODEL"],
296 + key=['IS_CAUSAL', 'dropout_p', 'BLOCK_DMODEL'],
297 )
298 @triton.jit
299 def attn_fwd(
@@ -330,8 +330,8 @@ def attn_fwd(
330 philox_seed,
331 philox_offset_base,
332 encoded_softmax,
333 - hq,
334 - hk,
333 + HQ: tl.constexpr,
334 + HK: tl.constexpr,
335 ACTUAL_BLOCK_DMODEL: tl.constexpr,
336 MAX_SEQLENS_Q: tl.constexpr,
337 MAX_SEQLENS_K: tl.constexpr,
@@ -403,7 +403,7 @@ def attn_fwd(
403 # We still need to write 0s to the result
404 # tl.store(O_block_ptr,
405 # acc.to(Out.type.element_ty), boundary_check=(0,1))
406 - # l_ptrs = L + off_z * hq * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q
406 + # l_ptrs = L + off_z * HQ * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q
407 # + offs_m
408 # We store inf to LSE, not -inf because in the bwd pass,
409 # we subtract this
@@ -414,11 +414,9 @@ def attn_fwd(
414 # TODO: Should dropout and return encoded softmax be handled here?
415 return
416
417 - is_mqa = hq != hk
418 - if is_mqa: # noqa: SIM108
419 - off_h_k = off_h_q % hk
420 - else:
421 - off_h_k = off_h_q
417 + # If MQA / GQA, set the K and V head offsets appropriately.
418 + GROUP_SIZE: tl.constexpr = HQ // HK
419 + off_h_k = off_h_q // GROUP_SIZE if GROUP_SIZE != 1 else off_h_q
420
421 n_extra_tokens = 0
422 if seqlen_k < BLOCK_N:
@@ -471,7 +469,7 @@ def attn_fwd(
469 bias_ptr = None
470 if ENABLE_DROPOUT:
471 batch_philox_offset = philox_offset_base \
474 - + (off_z * hq + off_h_q) \
472 + + (off_z * HQ + off_h_q) \
473 * seqlen_q * seqlen_k
474 else:
475 batch_philox_offset = 0
@@ -624,7 +622,7 @@ def attn_fwd(
622 z = 0.0
623 acc = tl.where(out_ptrs_mask, acc, z.to(acc.type.element_ty))
624 # write back LSE
627 - # l_ptrs = L + off_z * hq * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q + offs_m
625 + # l_ptrs = L + off_z * HQ * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q + offs_m
626 # If seqlen_q not multiple of BLOCK_M, we need to mask out the last
627 # few rows. This is only true for the last M block. For others,
628 # overflow_size will be -ve
@@ -784,8 +782,8 @@ class _attention(torch.autograd.Function):
782 philox_seed=philox_seed,
783 philox_offset_base=philox_offset,
784 encoded_softmax=encoded_softmax,
787 - hq=nheads_q,
788 - hk=nheads_k,
785 + HQ=nheads_q,
786 + HK=nheads_k,
787 ACTUAL_BLOCK_DMODEL=head_size,
788 MAX_SEQLENS_Q=max_seqlens_q,
789 MAX_SEQLENS_K=max_seqlens_k,