A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[ROCm][Hardware][AMD] Enable group query attention for triton FA (#4406)
Hongxia Yang committed
Apr 27, 2024 at 02:37 UTC
18d23f642af9c56f45ccf684a22f386fc54c6ead
2 files changed
+36
-41
vllm/attention/backends/rocm_flash_attn.py
+25
-28
@@ -253,36 +253,31 @@ class ROCmFlashAttentionImpl(AttentionImpl):
253
# triton attention
254
# When block_tables are not filled, it means q and k are the
255
# prompt, and they have the same length.
256
- if self.use_triton_flash_attn or self.use_naive_attn:
256
+ if self.use_triton_flash_attn:
257
+ out, _ = self.attn_func(
258
+ query,
259
+ key,
260
+ value,
261
+ None,
262
+ prefill_meta.seq_start_loc,
263
+ prefill_meta.seq_start_loc,
264
+ prefill_meta.max_prompt_len,
265
+ prefill_meta.max_prompt_len,
266
+ True,
267
+ self.scale,
268
+ )
269
+ elif self.use_naive_attn:
270
if self.num_kv_heads != self.num_heads:
271
# Interleave for MQA workaround.
272
key = self.repeat_kv(key, self.num_queries_per_kv)
273
value = self.repeat_kv(value, self.num_queries_per_kv)
261
- if self.use_naive_attn:
262
- out = self.attn_func(
263
- query,
264
- key,
265
- value,
266
- prefill_meta.prompt_lens,
267
- self.scale,
268
- )
269
- assert output[:num_prefill_tokens].shape == out.shape
270
- output[:num_prefill_tokens] = out
271
- else:
272
- out, _ = self.attn_func(
273
- query,
274
- key,
275
- value,
276
- None,
277
- prefill_meta.seq_start_loc,
278
- prefill_meta.seq_start_loc,
279
- prefill_meta.max_prompt_len,
280
- prefill_meta.max_prompt_len,
281
- True,
282
- self.scale,
283
- )
284
- assert output[:num_prefill_tokens].shape == out.shape
285
- output[:num_prefill_tokens] = out
274
+ out = self.attn_func(
275
+ query,
276
+ key,
277
+ value,
278
+ prefill_meta.prompt_lens,
279
+ self.scale,
280
+ )
281
else:
282
out = self.attn_func(
283
q=query,
@@ -295,8 +290,10 @@ class ROCmFlashAttentionImpl(AttentionImpl):
290
softmax_scale=self.scale,
291
causal=True,
292
)
298
- assert output[:num_prefill_tokens].shape == out.shape
299
- output[:num_prefill_tokens] = out
293
+
294
+ # common code for prefill
295
+ assert output[:num_prefill_tokens].shape == out.shape
296
+ output[:num_prefill_tokens] = out
297
else:
298
# prefix-enabled attention
299
output[:num_prefill_tokens] = PagedAttention.forward_prefix(
vllm/attention/ops/triton_flash_attention.py
+11
-13
@@ -293,7 +293,7 @@ def _attn_fwd_inner(
293
num_warps=4,
294
),
295
],
296
- key=["hq", "hk", "IS_CAUSAL", "dropout_p", "BLOCK_DMODEL"],
296
+ key=['IS_CAUSAL', 'dropout_p', 'BLOCK_DMODEL'],
297
)
298
@triton.jit
299
def attn_fwd(
@@ -330,8 +330,8 @@ def attn_fwd(
330
philox_seed,
331
philox_offset_base,
332
encoded_softmax,
333
- hq,
334
- hk,
333
+ HQ: tl.constexpr,
334
+ HK: tl.constexpr,
335
ACTUAL_BLOCK_DMODEL: tl.constexpr,
336
MAX_SEQLENS_Q: tl.constexpr,
337
MAX_SEQLENS_K: tl.constexpr,
@@ -403,7 +403,7 @@ def attn_fwd(
403
# We still need to write 0s to the result
404
# tl.store(O_block_ptr,
405
# acc.to(Out.type.element_ty), boundary_check=(0,1))
406
- # l_ptrs = L + off_z * hq * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q
406
+ # l_ptrs = L + off_z * HQ * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q
407
# + offs_m
408
# We store inf to LSE, not -inf because in the bwd pass,
409
# we subtract this
@@ -414,11 +414,9 @@ def attn_fwd(
414
# TODO: Should dropout and return encoded softmax be handled here?
415
return
416
417
- is_mqa = hq != hk
418
- if is_mqa: # noqa: SIM108
419
- off_h_k = off_h_q % hk
420
- else:
421
- off_h_k = off_h_q
417
+ # If MQA / GQA, set the K and V head offsets appropriately.
418
+ GROUP_SIZE: tl.constexpr = HQ // HK
419
+ off_h_k = off_h_q // GROUP_SIZE if GROUP_SIZE != 1 else off_h_q
420
421
n_extra_tokens = 0
422
if seqlen_k < BLOCK_N:
@@ -471,7 +469,7 @@ def attn_fwd(
469
bias_ptr = None
470
if ENABLE_DROPOUT:
471
batch_philox_offset = philox_offset_base \
474
- + (off_z * hq + off_h_q) \
472
+ + (off_z * HQ + off_h_q) \
473
* seqlen_q * seqlen_k
474
else:
475
batch_philox_offset = 0
@@ -624,7 +622,7 @@ def attn_fwd(
622
z = 0.0
623
acc = tl.where(out_ptrs_mask, acc, z.to(acc.type.element_ty))
624
# write back LSE
627
- # l_ptrs = L + off_z * hq * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q + offs_m
625
+ # l_ptrs = L + off_z * HQ * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q + offs_m
626
# If seqlen_q not multiple of BLOCK_M, we need to mask out the last
627
# few rows. This is only true for the last M block. For others,
628
# overflow_size will be -ve
@@ -784,8 +782,8 @@ class _attention(torch.autograd.Function):
782
philox_seed=philox_seed,
783
philox_offset_base=philox_offset,
784
encoded_softmax=encoded_softmax,
787
- hq=nheads_q,
788
- hk=nheads_k,
785
+ HQ=nheads_q,
786
+ HK=nheads_k,
787
ACTUAL_BLOCK_DMODEL=head_size,
788
MAX_SEQLENS_Q=max_seqlens_q,
789
MAX_SEQLENS_K=max_seqlens_k,