A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[ROCm][Hardware][AMD] Use Triton Kernel for default FA on ROCm (#3643)
Co-authored-by: jpvillam <jpvillam@amd.com> Co-authored-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Juan Villamizar committed
Apr 9, 2024 at 17:10 UTC
6c0b04515fee7b402a6febde1467581825bb2164
5 files changed
+1213
-93
Dockerfile.rocm
+14
@@ -23,6 +23,9 @@ RUN echo "FA_BRANCH is $FA_BRANCH"
23
# In that case, we need to use the python reference attention implementation in vllm
24
ARG BUILD_FA="1"
25
26
+# whether to build triton on rocm
27
+ARG BUILD_TRITON="1"
28
+
29
# Install some basic utilities
30
RUN apt-get update && apt-get install python3 python3-pip -y
31
@@ -75,6 +78,17 @@ RUN if [ "$BUILD_FA" = "1" ]; then \
78
RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" ]; then \
79
rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
80
81
+# build triton
82
+RUN if [ "$BUILD_TRITON" = "1" ]; then \
83
+ mkdir -p libs \
84
+ && cd libs \
85
+ && pip uninstall -y triton \
86
+ && git clone https://github.com/ROCm/triton.git \
87
+ && cd triton/python \
88
+ && pip3 install . \
89
+ && cd ../..; \
90
+ fi
91
+
92
COPY ./ /app/vllm
93
94
RUN python3 -m pip install --upgrade pip
vllm/attention/backends/rocm_flash_attn.py
new
+348
@@ -0,0 +1,348 @@
1
+"""Attention layer ROCm GPUs."""
2
+import os
3
+from dataclasses import dataclass
4
+from typing import Dict, List, Optional, Tuple, Type
5
+
6
+import torch
7
+
8
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9
+ AttentionMetadata)
10
+from vllm.attention.ops.paged_attn import (PagedAttention,
11
+ PagedAttentionMetadata)
12
+from vllm.logger import init_logger
13
+
14
+logger = init_logger(__name__)
15
+
16
+
17
+class ROCmFlashAttentionBackend(AttentionBackend):
18
+
19
+ @staticmethod
20
+ def get_impl_cls() -> Type["ROCmFlashAttentionImpl"]:
21
+ return ROCmFlashAttentionImpl
22
+
23
+ @staticmethod
24
+ def make_metadata(*args, **kwargs) -> "ROCmFlashAttentionMetadata":
25
+ return ROCmFlashAttentionMetadata(*args, **kwargs)
26
+
27
+ @staticmethod
28
+ def get_kv_cache_shape(
29
+ num_blocks: int,
30
+ block_size: int,
31
+ num_kv_heads: int,
32
+ head_size: int,
33
+ ) -> Tuple[int, ...]:
34
+ return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
35
+ num_kv_heads, head_size)
36
+
37
+ @staticmethod
38
+ def swap_blocks(
39
+ src_kv_cache: torch.Tensor,
40
+ dst_kv_cache: torch.Tensor,
41
+ src_to_dst: Dict[int, int],
42
+ ) -> None:
43
+ PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
44
+
45
+ @staticmethod
46
+ def copy_blocks(
47
+ kv_caches: List[torch.Tensor],
48
+ src_to_dists: Dict[int, List[int]],
49
+ ) -> None:
50
+ PagedAttention.copy_blocks(kv_caches, src_to_dists)
51
+
52
+
53
+@dataclass
54
+class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
55
+ """Metadata for FlashAttentionBackend.
56
+
57
+ NOTE: Any python object stored here is not updated when it is
58
+ cuda-graph replayed. If you have values that need to be changed
59
+ dynamically, it should be stored in tensor. The tensor has to be
60
+ updated from `CUDAGraphRunner.forward` API.
61
+ """
62
+ # Currently, input sequences can only contain all prompts
63
+ # or all decoding. True if all sequences are prompts.
64
+ is_prompt: bool
65
+ # (batch_size,). The prompt length per sequence. None if it is a decoding.
66
+ prompt_lens: Optional[List[int]]
67
+ # prompt_lens stored as a tensor.
68
+ prompt_lens_tensor: Optional[torch.Tensor]
69
+ # The number of prompt tokens. Doesn't include padding.
70
+ num_prompt_tokens: int
71
+ # The number of generation tokens. Doesn't include padding.
72
+ num_generation_tokens: int
73
+
74
+ # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75
+ # |---------- N-1 iteration --------|
76
+ # |---------------- N iteration ---------------------|
77
+ # |- tokenA -|......................|-- newTokens ---|
78
+ # |---------- context_len ----------|
79
+ # |-------------------- seqlen ----------------------|
80
+ # |- subquery_len -|
81
+
82
+ # WARNING(sang): context_len has different definition depending on if it is
83
+ # prefill vs decoding. When it is prefill, it doesn't include new tokens.
84
+ # When it is for decoding, it includes a new token.
85
+
86
+ # Maximum subquery length in the batch.
87
+ max_subquery_len: Optional[int]
88
+ # Maximum prompt length in the batch.
89
+ max_prompt_len: Optional[int]
90
+ # (batch_size + 1,). The cumulative subquery lengths of the sequences in
91
+ # the batch, used to index into subquery. E.g., if the subquery length
92
+ # is [4, 6], it is [0, 4, 10].
93
+ subquery_start_loc: Optional[torch.Tensor]
94
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
95
+ # the batch, used to index into sequence. E.g., if the sequence length is
96
+ # [4, 6], it is [0, 4, 10].
97
+ seq_start_loc: Optional[torch.Tensor]
98
+
99
+ # Whether or not if cuda graph is enabled.
100
+ # Cuda-graph is currently enabled for decoding only.
101
+ # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
102
+ use_cuda_graph: bool
103
+
104
+
105
+class ROCmFlashAttentionImpl(AttentionImpl):
106
+ """
107
+ If the input tensors contain prompt tokens, the layout is as follows:
108
+ |<--------------- num_prompt_tokens -------------->|
109
+ |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
110
+
111
+ Otherwise, the layout is as follows:
112
+ |<------------------ num_generation_tokens (M) ----------------->|
113
+ |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
114
+
115
+ Generation tokens can contain padding when cuda-graph is used.
116
+ Currently, prompt tokens don't contain any padding.
117
+
118
+ The prompts might have different lengths, while the generation tokens
119
+ always have length 1.
120
+ """
121
+
122
+ def __init__(
123
+ self,
124
+ num_heads: int,
125
+ head_size: int,
126
+ scale: float,
127
+ num_kv_heads: Optional[int] = None,
128
+ alibi_slopes: Optional[List[float]] = None,
129
+ sliding_window: Optional[int] = None,
130
+ ) -> None:
131
+ self.num_heads = num_heads
132
+ self.head_size = head_size
133
+ self.scale = float(scale)
134
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
135
+ self.sliding_window = ((sliding_window, sliding_window)
136
+ if sliding_window is not None else (-1, -1))
137
+ if alibi_slopes is not None:
138
+ alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
139
+ self.alibi_slopes = alibi_slopes
140
+
141
+ assert self.num_heads % self.num_kv_heads == 0
142
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
143
+
144
+ suppored_head_sizes = PagedAttention.get_supported_head_sizes()
145
+ if head_size not in suppored_head_sizes:
146
+ raise ValueError(
147
+ f"Head size {head_size} is not supported by PagedAttention. "
148
+ f"Supported head sizes are: {suppored_head_sizes}.")
149
+
150
+ self.use_naive_attn = torch.cuda.get_device_capability()[0] != 9
151
+ # NOTE: Allow for switching between Triton and CK. Defaulting to triton.
152
+ self.use_triton_flash_attn = (os.environ.get(
153
+ "VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in ("true", "1"))
154
+ if self.use_naive_attn:
155
+ # AMD Radeon 7900 series (gfx1100) currently does not support
156
+ # xFormers nor FlashAttention. As a temporary workaround, we use
157
+ # naive PyTorch implementation of attention.
158
+ self.attn_fuc = _naive_attention()
159
+ logger.debug("Using naive attention in ROCmBackend")
160
+ elif self.use_triton_flash_attn:
161
+ from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
162
+ triton_attention)
163
+ self.attn_func = triton_attention
164
+ logger.debug("Using Triton FA in ROCmBackend")
165
+ else:
166
+ from flash_attn import flash_attn_varlen_func # noqa: F401
167
+ self.attn_func = flash_attn_varlen_func
168
+ logger.debug("Using CK FA in ROCmBackend")
169
+
170
+ def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
171
+ """torch.repeat_interleave(x, dim=1, repeats=n_rep)"""
172
+ tokens, n_kv_heads, head_dim = x.shape
173
+ return (x[:, :,
174
+ None, :].expand(tokens, n_kv_heads, n_rep,
175
+ head_dim).reshape(tokens, n_kv_heads * n_rep,
176
+ head_dim))
177
+
178
+ def forward(
179
+ self,
180
+ query: torch.Tensor,
181
+ key: torch.Tensor,
182
+ value: torch.Tensor,
183
+ kv_cache: torch.Tensor,
184
+ attn_metadata: ROCmFlashAttentionMetadata,
185
+ kv_scale: float = 1.0,
186
+ ) -> torch.Tensor:
187
+ """Forward pass with FlashAttention and PagedAttention.
188
+
189
+ Args:
190
+ query: shape = [num_tokens, num_heads * head_size]
191
+ key: shape = [num_tokens, num_kv_heads * head_size]
192
+ value: shape = [num_tokens, num_kv_heads * head_size]
193
+ kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
194
+ attn_metadata: Metadata for attention.
195
+ Returns:
196
+ shape = [num_tokens, num_heads * head_size]
197
+ """
198
+ num_tokens, hidden_size = query.shape
199
+ # Reshape the query, key, and value tensors.
200
+ query = query.view(-1, self.num_heads, self.head_size)
201
+ key = key.view(-1, self.num_kv_heads, self.head_size)
202
+ value = value.view(-1, self.num_kv_heads, self.head_size)
203
+
204
+ if kv_cache is not None:
205
+ key_cache, value_cache = PagedAttention.split_kv_cache(
206
+ kv_cache, self.num_kv_heads, self.head_size)
207
+
208
+ # Reshape the input keys and values and store them in the cache.
209
+ # If kv_cache is not provided, the new key and value tensors are
210
+ # not cached. This happens during the initial memory profiling run.
211
+ PagedAttention.write_to_paged_cache(
212
+ key,
213
+ value,
214
+ key_cache,
215
+ value_cache,
216
+ attn_metadata.slot_mapping,
217
+ attn_metadata.kv_cache_dtype,
218
+ kv_scale,
219
+ )
220
+
221
+ if attn_metadata.is_prompt:
222
+ # Prompt run.
223
+ if kv_cache is None or attn_metadata.block_tables.numel() == 0:
224
+ # triton attention
225
+ # When block_tables are not filled, it means q and k are the
226
+ # prompt, and they have the same length.
227
+ if self.use_naive_attn or self.use_triton_flash_attn:
228
+ if self.num_kv_heads != self.num_heads:
229
+ # Interleave for MQA workaround.
230
+ key = self.repeat_kv(key, self.num_queries_per_kv)
231
+ value = self.repeat_kv(value, self.num_queries_per_kv)
232
+ if self.use_naive_attn:
233
+ output = self.attn_fuc(
234
+ query,
235
+ key,
236
+ value,
237
+ attn_metadata.prompt_lens,
238
+ self.scale,
239
+ )
240
+ else:
241
+ output, _ = self.attn_func(
242
+ query,
243
+ key,
244
+ value,
245
+ None,
246
+ attn_metadata.seq_start_loc,
247
+ attn_metadata.seq_start_loc,
248
+ attn_metadata.max_prompt_len,
249
+ attn_metadata.max_prompt_len,
250
+ True,
251
+ self.scale,
252
+ )
253
+ else:
254
+ output = self.attn_func(
255
+ q=query,
256
+ k=key,
257
+ v=value,
258
+ cu_seqlens_q=attn_metadata.seq_start_loc,
259
+ cu_seqlens_k=attn_metadata.seq_start_loc,
260
+ max_seqlen_q=attn_metadata.max_prompt_len,
261
+ max_seqlen_k=attn_metadata.max_prompt_len,
262
+ softmax_scale=self.scale,
263
+ causal=True,
264
+ )
265
+
266
+ else:
267
+ # prefix-enabled attention
268
+ output = PagedAttention.forward_prefix(
269
+ query,
270
+ key,
271
+ value,
272
+ key_cache,
273
+ value_cache,
274
+ attn_metadata.block_tables,
275
+ attn_metadata.subquery_start_loc,
276
+ attn_metadata.prompt_lens_tensor,
277
+ attn_metadata.context_lens,
278
+ attn_metadata.max_subquery_len,
279
+ self.alibi_slopes,
280
+ )
281
+ else:
282
+ # Decoding run.
283
+ output = PagedAttention.forward_decode(
284
+ query,
285
+ key_cache,
286
+ value_cache,
287
+ attn_metadata.block_tables,
288
+ attn_metadata.context_lens,
289
+ attn_metadata.max_context_len,
290
+ attn_metadata.kv_cache_dtype,
291
+ self.num_kv_heads,
292
+ self.scale,
293
+ self.alibi_slopes,
294
+ kv_scale,
295
+ )
296
+
297
+ # Reshape the output tensor.
298
+ return output.view(num_tokens, hidden_size)
299
+
300
+
301
+def _naive_attention(
302
+ query: torch.Tensor,
303
+ key: torch.Tensor,
304
+ value: torch.Tensor,
305
+ prompt_lens: List[int],
306
+ scale: float,
307
+) -> torch.Tensor:
308
+ num_tokens = query.shape[0]
309
+ output = torch.empty_like(query)
310
+ start = 0
311
+ for _, prompt_len in enumerate(prompt_lens):
312
+ end = start + prompt_len
313
+ out = _naive_masked_attention(
314
+ query[None, start:end],
315
+ key[None, start:end],
316
+ value[None, start:end],
317
+ scale,
318
+ )
319
+ # TODO(woosuk): Unnecessary copy. Optimize.
320
+ output[start:end].copy_(out)
321
+ start += prompt_len
322
+
323
+ # Using view got RuntimeError: view size is not compatible
324
+ # with input tensor's size and stride (at least one
325
+ # dimension spans across two contiguous subspaces).
326
+ # Use reshape instead.
327
+ return output.reshape(num_tokens, -1)
328
+
329
+
330
+def _naive_masked_attention(
331
+ query: torch.Tensor,
332
+ key: torch.Tensor,
333
+ value: torch.Tensor,
334
+ scale: float,
335
+) -> torch.Tensor:
336
+ seq_len, _, _ = query.shape
337
+ attn_mask = torch.triu(torch.ones(seq_len,
338
+ seq_len,
339
+ dtype=query.dtype,
340
+ device=query.device),
341
+ diagonal=1)
342
+ attn_mask = attn_mask * torch.finfo(query.dtype).min
343
+
344
+ attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
345
+ attn_weights = attn_weights + attn_mask.float()
346
+ attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
347
+ out = torch.einsum("hqk,khd->qhd", attn_weights, value)
348
+ return out
vllm/attention/backends/xformers.py
+2
-76
@@ -1,5 +1,4 @@
1
"""Attention layer with xFormers and PagedAttention."""
2
-import importlib
2
from dataclasses import dataclass
3
from typing import Dict, List, Optional, Tuple, Type
4
@@ -14,7 +13,6 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13
from vllm.attention.ops.paged_attn import (PagedAttention,
14
PagedAttentionMetadata)
15
from vllm.logger import init_logger
17
-from vllm.utils import is_hip
16
17
logger = init_logger(__name__)
18
@@ -166,11 +164,6 @@ class XFormersImpl(AttentionImpl):
164
f"Head size {head_size} is not supported by PagedAttention. "
165
f"Supported head sizes are: {suppored_head_sizes}.")
166
169
- # AMD Radeon 7900 series (gfx1100) currently does not support xFormers
170
- # nor FlashAttention. As a temporary workaround, we use naive PyTorch
171
- # implementation of attention.
172
- self.use_naive_attention = _check_use_naive_attention()
173
-
167
def forward(
168
self,
169
query: torch.Tensor,
@@ -233,30 +226,6 @@ class XFormersImpl(AttentionImpl):
226
self.num_queries_per_kv,
227
value.shape[-1])
228
236
- if self.use_naive_attention:
237
- output = torch.empty_like(query)
238
- start = 0
239
- for _, prompt_len in enumerate(attn_metadata.prompt_lens):
240
- end = start + prompt_len
241
- out = _naive_masked_attention(
242
- query[None, start:end],
243
- key[None, start:end],
244
- value[None, start:end],
245
- self.num_heads,
246
- self.num_kv_heads,
247
- self.head_size,
248
- self.scale,
249
- )
250
- # TODO(woosuk): Unnecessary copy. Optimize.
251
- output[start:end].copy_(out)
252
- start += prompt_len
253
-
254
- # Using view got RuntimeError: view size is not compatible
255
- # with input tensor's size and stride (at least one
256
- # dimension spans across two contiguous subspaces).
257
- # Use reshape instead.
258
- return output.reshape(num_tokens, hidden_size)
259
-
229
output = self._run_memory_efficient_xformers_forward(
230
query, key, value, attn_metadata)
231
else:
@@ -329,8 +298,6 @@ class XFormersImpl(AttentionImpl):
298
self.alibi_slopes, self.num_kv_heads, query.dtype,
299
attn_metadata.prompt_lens)
300
332
- op = xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if (
333
- is_hip()) else None
301
# No alibi slopes.
302
# TODO(woosuk): Too many view operations. Let's try to reduce
303
# them in the future for code readability.
@@ -344,8 +311,7 @@ class XFormersImpl(AttentionImpl):
311
value,
312
attn_bias=attn_metadata.attn_bias[0],
313
p=0.0,
347
- scale=self.scale,
348
- op=op)
314
+ scale=self.scale)
315
316
return out.view_as(query)
317
@@ -363,8 +329,7 @@ class XFormersImpl(AttentionImpl):
329
value[None, start:end],
330
attn_bias=attn_metadata.attn_bias[i],
331
p=0.0,
366
- scale=self.scale,
367
- op=op)
332
+ scale=self.scale)
333
# TODO(woosuk): Unnecessary copy. Optimize.
334
output[start:end].copy_(out.squeeze(0))
335
start += prompt_len
@@ -405,42 +370,3 @@ def _make_alibi_bias(
370
attn_biases.append(LowerTriangularMaskWithTensorBias(bias))
371
372
return attn_biases
408
-
409
-
410
-def _check_use_naive_attention() -> bool:
411
- if not is_hip():
412
- return False
413
- # For ROCm, check whether flash attention is installed or not.
414
- use_naive_attention = importlib.util.find_spec("flash_attn") is None
415
- if use_naive_attention:
416
- logger.warning("flash_attn is not installed. Using naive attention. "
417
- "This will take significantly more GPU memory.")
418
- return True
419
- return False
420
-
421
-
422
-def _naive_masked_attention(
423
- query: torch.Tensor,
424
- key: torch.Tensor,
425
- value: torch.Tensor,
426
- num_heads: int,
427
- num_kv_heads: int,
428
- head_size: int,
429
- scale: float,
430
-) -> torch.Tensor:
431
- query = query.view(-1, num_heads, head_size)
432
- key = key.view(-1, num_kv_heads, head_size)
433
- value = value.view(-1, num_kv_heads, head_size)
434
- seq_len, _, _ = query.shape
435
- attn_mask = torch.triu(torch.ones(seq_len,
436
- seq_len,
437
- dtype=query.dtype,
438
- device=query.device),
439
- diagonal=1)
440
- attn_mask = attn_mask * torch.finfo(query.dtype).min
441
-
442
- attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
443
- attn_weights = attn_weights + attn_mask.float()
444
- attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
445
- out = torch.einsum("hqk,khd->qhd", attn_weights, value)
446
- return out
vllm/attention/ops/triton_flash_attention.py
new
+809
@@ -0,0 +1,809 @@
1
+#!/usr/bin/env python
2
+"""
3
+Fused Attention
4
+===============
5
+
6
+This is a Triton implementation of the Flash Attention v2 algorithm from Tri Dao
7
+(https://tridao.me/publications/flash2/flash2.pdf)
8
+Credits: OpenAI kernel team, AMD ML Frameworks Triton team
9
+
10
+Features supported:
11
+
12
+1) Fwd with causal masking
13
+2) Any sequence lengths without padding (currently fwd kernel only)
14
+3) Support for different sequence lengths for q and k
15
+4) Nested tensor API currently does not support dropout or bias.
16
+
17
+Not currently supported:
18
+
19
+1) Non power of two head dims
20
+
21
+"""
22
+
23
+import torch
24
+import triton
25
+import triton.language as tl
26
+
27
+torch_dtype: tl.constexpr = torch.float16
28
+
29
+
30
+@triton.jit
31
+def cdiv_fn(x, y):
32
+ return (x + y - 1) // y
33
+
34
+
35
+@triton.jit
36
+def max_fn(x, y):
37
+ return tl.math.max(x, y)
38
+
39
+
40
+@triton.jit
41
+def dropout_offsets(philox_seed, philox_offset, dropout_p, m, n, stride):
42
+ ms = tl.arange(0, m)
43
+ ns = tl.arange(0, n)
44
+ return philox_offset + ms[:, None] * stride + ns[None, :]
45
+
46
+
47
+@triton.jit
48
+def dropout_rng(philox_seed, philox_offset, dropout_p, m, n, stride):
49
+ rng_offsets = dropout_offsets(philox_seed, philox_offset, dropout_p, m, n,
50
+ stride).to(tl.uint32)
51
+ # TODO: use tl.randint for better performance
52
+ return tl.rand(philox_seed, rng_offsets)
53
+
54
+
55
+@triton.jit
56
+def dropout_mask(philox_seed, philox_offset, dropout_p, m, n, stride):
57
+ rng_output = dropout_rng(philox_seed, philox_offset, dropout_p, m, n,
58
+ stride)
59
+ rng_keep = rng_output > dropout_p
60
+ return rng_keep
61
+
62
+
63
+@triton.jit
64
+def load_fn(block_ptr, first, second, pad):
65
+ if first and second:
66
+ tensor = tl.load(block_ptr, boundary_check=(0, 1), padding_option=pad)
67
+ elif first:
68
+ tensor = tl.load(block_ptr, boundary_check=(0, ), padding_option=pad)
69
+ elif second:
70
+ tensor = tl.load(block_ptr, boundary_check=(1, ), padding_option=pad)
71
+ else:
72
+ tensor = tl.load(block_ptr)
73
+ return tensor
74
+
75
+
76
+@triton.jit
77
+def _attn_fwd_inner(
78
+ acc,
79
+ l_i,
80
+ m_i,
81
+ q,
82
+ K_block_ptr,
83
+ V_block_ptr,
84
+ start_m,
85
+ actual_seqlen_k,
86
+ dropout_p,
87
+ philox_seed,
88
+ batch_philox_offset,
89
+ encoded_softmax_block_ptr,
90
+ block_min,
91
+ block_max,
92
+ offs_n_causal,
93
+ masked_blocks,
94
+ n_extra_tokens,
95
+ bias_ptr,
96
+ IS_CAUSAL: tl.constexpr,
97
+ BLOCK_M: tl.constexpr,
98
+ BLOCK_DMODEL: tl.constexpr,
99
+ BLOCK_N: tl.constexpr,
100
+ OFFS_M: tl.constexpr,
101
+ OFFS_N: tl.constexpr,
102
+ PRE_LOAD_V: tl.constexpr,
103
+ MASK_STEPS: tl.constexpr,
104
+ ENABLE_DROPOUT: tl.constexpr,
105
+ RETURN_ENCODED_SOFTMAX: tl.constexpr,
106
+ PADDED_HEAD: tl.constexpr,
107
+):
108
+ # loop over k, v, and update accumulator
109
+ for start_n in range(block_min, block_max, BLOCK_N):
110
+ # For padded blocks, we will overrun the tensor size if
111
+ # we load all BLOCK_N. For others, the blocks are all within range.
112
+ k = load_fn(
113
+ K_block_ptr,
114
+ PADDED_HEAD,
115
+ MASK_STEPS and (n_extra_tokens != 0),
116
+ "zero",
117
+ )
118
+ if PRE_LOAD_V:
119
+ v = load_fn(
120
+ V_block_ptr,
121
+ MASK_STEPS and (n_extra_tokens != 0),
122
+ PADDED_HEAD,
123
+ "zero",
124
+ )
125
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
126
+ # We start from end of seqlen_k so only the first iteration would need
127
+ # to be checked for padding if it is not a multiple of block_n
128
+ # TODO: This can be optimized to only be true for the padded block.
129
+ if MASK_STEPS: # noqa: SIM102
130
+ # If this is the last block / iteration, we want to
131
+ # mask if the sequence length is not a multiple of block size
132
+ # a solution is to always do BLOCK_M // BLOCK_N + 1 steps
133
+ # if not is_modulo_mn. last step might get wasted but that is okay.
134
+ # check if this masking works for that case.
135
+ if (start_n + BLOCK_N == block_max) and (n_extra_tokens != 0):
136
+ boundary_m = tl.full([BLOCK_M],
137
+ actual_seqlen_k,
138
+ dtype=tl.int32)
139
+ size_n = start_n + OFFS_N[None, :]
140
+ mask = size_n < boundary_m[:, None]
141
+ qk = tl.where(mask, qk, float("-inf"))
142
+ if IS_CAUSAL:
143
+ causal_boundary = start_n + offs_n_causal
144
+ causal_mask = OFFS_M[:, None] >= causal_boundary[None, :]
145
+ qk = tl.where(causal_mask, qk, float("-inf"))
146
+ # -- compute qk ----
147
+ qk += tl.dot(q, k)
148
+ if bias_ptr is not None:
149
+ bias = load_fn(bias_ptr, False, MASK_STEPS
150
+ and (n_extra_tokens != 0), "zero")
151
+ # While bias is added after multiplying qk with sm_scale, our
152
+ # optimization to use 2^x instead of e^x results in an additional
153
+ # scale factor of log2(e) which we must also multiply the bias with.
154
+ qk += bias * 1.44269504089
155
+ m_ij = tl.maximum(m_i, tl.max(qk, 1))
156
+ qk = qk - m_ij[:, None]
157
+ p = tl.math.exp2(qk)
158
+
159
+ # CAVEAT: Must update l_ij before applying dropout
160
+ l_ij = tl.sum(p, 1)
161
+ if ENABLE_DROPOUT:
162
+ philox_offset = (batch_philox_offset +
163
+ start_m * BLOCK_M * actual_seqlen_k + start_n -
164
+ BLOCK_N)
165
+ keep = dropout_mask(
166
+ philox_seed,
167
+ philox_offset,
168
+ dropout_p,
169
+ BLOCK_M,
170
+ BLOCK_N,
171
+ actual_seqlen_k,
172
+ )
173
+ if RETURN_ENCODED_SOFTMAX:
174
+ tl.store(
175
+ encoded_softmax_block_ptr,
176
+ tl.where(keep, p,
177
+ -p).to(encoded_softmax_block_ptr.type.element_ty),
178
+ )
179
+ p = tl.where(keep, p, 0.0)
180
+ elif RETURN_ENCODED_SOFTMAX:
181
+ tl.store(
182
+ encoded_softmax_block_ptr,
183
+ p.to(encoded_softmax_block_ptr.type.element_ty),
184
+ )
185
+ # -- update output accumulator --
186
+ alpha = tl.math.exp2(m_i - m_ij)
187
+ acc = acc * alpha[:, None]
188
+ if not PRE_LOAD_V:
189
+ v = load_fn(
190
+ V_block_ptr,
191
+ MASK_STEPS and (n_extra_tokens != 0),
192
+ PADDED_HEAD,
193
+ "zero",
194
+ )
195
+ # -- update m_i and l_i
196
+ l_i = l_i * alpha + l_ij
197
+ # update m_i and l_i
198
+ m_i = m_ij
199
+ acc += tl.dot(p.to(V_block_ptr.type.element_ty), v)
200
+ V_block_ptr = tl.advance(V_block_ptr, (BLOCK_N, 0))
201
+ K_block_ptr = tl.advance(K_block_ptr, (0, BLOCK_N))
202
+ if bias_ptr is not None:
203
+ bias_ptr = tl.advance(bias_ptr, (0, BLOCK_N))
204
+ if RETURN_ENCODED_SOFTMAX:
205
+ encoded_softmax_block_ptr = tl.advance(encoded_softmax_block_ptr,
206
+ (0, BLOCK_N))
207
+ return acc, l_i, m_i
208
+
209
+
210
+@triton.autotune(
211
+ configs=[
212
+ triton.Config(
213
+ {
214
+ "BLOCK_M": 256,
215
+ "BLOCK_N": 64,
216
+ "waves_per_eu": 2,
217
+ "PRE_LOAD_V": False,
218
+ },
219
+ num_stages=1,
220
+ num_warps=8,
221
+ ),
222
+ triton.Config(
223
+ {
224
+ "BLOCK_M": 128,
225
+ "BLOCK_N": 128,
226
+ "waves_per_eu": 2,
227
+ "PRE_LOAD_V": False,
228
+ },
229
+ num_stages=1,
230
+ num_warps=4,
231
+ ),
232
+ triton.Config(
233
+ {
234
+ "BLOCK_M": 256,
235
+ "BLOCK_N": 128,
236
+ "waves_per_eu": 2,
237
+ "PRE_LOAD_V": False,
238
+ },
239
+ num_stages=1,
240
+ num_warps=8,
241
+ ),
242
+ triton.Config(
243
+ {
244
+ "BLOCK_M": 128,
245
+ "BLOCK_N": 64,
246
+ "waves_per_eu": 3,
247
+ "PRE_LOAD_V": True,
248
+ },
249
+ num_stages=1,
250
+ num_warps=4,
251
+ ),
252
+ triton.Config(
253
+ {
254
+ "BLOCK_M": 128,
255
+ "BLOCK_N": 64,
256
+ "waves_per_eu": 3,
257
+ "PRE_LOAD_V": False,
258
+ },
259
+ num_stages=1,
260
+ num_warps=4,
261
+ ),
262
+ triton.Config(
263
+ {
264
+ "BLOCK_M": 64,
265
+ "BLOCK_N": 64,
266
+ "waves_per_eu": 4,
267
+ "PRE_LOAD_V": False,
268
+ },
269
+ num_stages=1,
270
+ num_warps=8,
271
+ ),
272
+ triton.Config(
273
+ {
274
+ "BLOCK_M": 32,
275
+ "BLOCK_N": 32,
276
+ "waves_per_eu": 4,
277
+ "PRE_LOAD_V": False,
278
+ },
279
+ num_stages=1,
280
+ num_warps=8,
281
+ ),
282
+ # TODO: This config fails with head_size not pow2 with data mismatches.
283
+ # triton.Config({'BLOCK_M': 32, 'BLOCK_N': 16, 'waves_per_eu': 1,
284
+ # 'PRE_LOAD_V': False}, num_stages=1, num_warps=4),
285
+ triton.Config(
286
+ {
287
+ "BLOCK_M": 16,
288
+ "BLOCK_N": 16,
289
+ "waves_per_eu": 1,
290
+ "PRE_LOAD_V": False,
291
+ },
292
+ num_stages=1,
293
+ num_warps=4,
294
+ ),
295
+ ],
296
+ key=["hq", "hk", "IS_CAUSAL", "dropout_p", "BLOCK_DMODEL"],
297
+)
298
+@triton.jit
299
+def attn_fwd(
300
+ Q,
301
+ K,
302
+ V,
303
+ bias,
304
+ sm_scale,
305
+ L,
306
+ Out,
307
+ stride_qz,
308
+ stride_qh,
309
+ stride_qm,
310
+ stride_qk,
311
+ stride_kz,
312
+ stride_kh,
313
+ stride_kn,
314
+ stride_kk,
315
+ stride_vz,
316
+ stride_vh,
317
+ stride_vk,
318
+ stride_vn,
319
+ stride_oz,
320
+ stride_oh,
321
+ stride_om,
322
+ stride_on,
323
+ stride_bz,
324
+ stride_bh,
325
+ stride_bm,
326
+ stride_bn,
327
+ cu_seqlens_q,
328
+ cu_seqlens_k,
329
+ dropout_p,
330
+ philox_seed,
331
+ philox_offset_base,
332
+ encoded_softmax,
333
+ hq,
334
+ hk,
335
+ ACTUAL_BLOCK_DMODEL: tl.constexpr,
336
+ MAX_SEQLENS_Q: tl.constexpr,
337
+ MAX_SEQLENS_K: tl.constexpr,
338
+ VARLEN: tl.constexpr,
339
+ IS_CAUSAL: tl.constexpr,
340
+ BLOCK_M: tl.constexpr,
341
+ BLOCK_DMODEL: tl.constexpr,
342
+ BLOCK_N: tl.constexpr,
343
+ PRE_LOAD_V: tl.constexpr,
344
+ BIAS_TYPE: tl.constexpr,
345
+ ENABLE_DROPOUT: tl.constexpr,
346
+ RETURN_ENCODED_SOFTMAX: tl.constexpr,
347
+):
348
+ start_m = tl.program_id(0)
349
+ off_h_q = tl.program_id(1)
350
+ off_z = tl.program_id(2)
351
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
352
+ offs_n = tl.arange(0, BLOCK_N)
353
+ if VARLEN:
354
+ cu_seqlens_q_start = tl.load(cu_seqlens_q + off_z)
355
+ cu_seqlens_q_end = tl.load(cu_seqlens_q + off_z + 1)
356
+ seqlen_q = cu_seqlens_q_end - cu_seqlens_q_start
357
+ # We have a one-size-fits-all grid in id(0). Some seqlens might be too
358
+ # small for all start_m so for those we return early.
359
+ if start_m * BLOCK_M > seqlen_q:
360
+ return
361
+ cu_seqlens_k_start = tl.load(cu_seqlens_k + off_z)
362
+ cu_seqlens_k_end = tl.load(cu_seqlens_k + off_z + 1)
363
+ seqlen_k = cu_seqlens_k_end - cu_seqlens_k_start
364
+ else:
365
+ cu_seqlens_q_start = 0
366
+ cu_seqlens_k_start = 0
367
+ seqlen_q = MAX_SEQLENS_Q
368
+ seqlen_k = MAX_SEQLENS_K
369
+
370
+ # Now we compute whether we need to exit early due to causal masking.
371
+ # This is because for seqlen_q > seqlen_k, M rows of the attn scores
372
+ # are completely masked, resulting in 0s written to the output, and
373
+ # inf written to LSE. We don't need to do any GEMMs in this case.
374
+ # This block of code determines what N is, and if this WG is operating
375
+ # on those M rows.
376
+ n_blocks = cdiv_fn(seqlen_k, BLOCK_N)
377
+ if IS_CAUSAL:
378
+ # If seqlen_q == seqlen_k, the attn scores are a square matrix.
379
+ # If seqlen_q != seqlen_k, attn scores are rectangular which means
380
+ # the causal mask boundary is bottom right aligned, and ends at either
381
+ # the top edge (seqlen_q < seqlen_k) or left edge.
382
+ # This captures the decrease in n_blocks if we have a rectangular attn
383
+ # matrix
384
+ n_blocks_seqlen = cdiv_fn(
385
+ (start_m + 1) * BLOCK_M + seqlen_k - seqlen_q, BLOCK_N)
386
+ # This is what adjusts the block_max for the current WG, only
387
+ # if IS_CAUSAL. Otherwise we want to always iterate through all n_blocks
388
+ n_blocks = min(n_blocks, n_blocks_seqlen)
389
+ # If we have no blocks after adjusting for seqlen deltas, this WG is
390
+ # part of the blocks that are all 0. We exit early.
391
+ if n_blocks <= 0:
392
+ o_offset = (off_z * stride_oz + cu_seqlens_q_start * stride_om +
393
+ off_h_q * stride_oh)
394
+ O_block_ptr = tl.make_block_ptr(
395
+ base=Out + o_offset,
396
+ shape=(seqlen_q, BLOCK_DMODEL),
397
+ strides=(stride_om, stride_on),
398
+ offsets=(start_m * BLOCK_M, 0),
399
+ block_shape=(BLOCK_M, BLOCK_DMODEL),
400
+ order=(1, 0),
401
+ )
402
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=Out.type.element_ty)
403
+ # We still need to write 0s to the result
404
+ # tl.store(O_block_ptr,
405
+ # acc.to(Out.type.element_ty), boundary_check=(0,1))
406
+ # l_ptrs = L + off_z * hq * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q
407
+ # + offs_m
408
+ # We store inf to LSE, not -inf because in the bwd pass,
409
+ # we subtract this
410
+ # from qk which makes it -inf, such that exp(qk - inf) = 0
411
+ # for these masked blocks.
412
+ # l = tl.full([BLOCK_M], value=float("inf"), dtype=tl.float32)
413
+ # tl.store(l_ptrs, l)
414
+ # TODO: Should dropout and return encoded softmax be handled here?
415
+ return
416
+
417
+ is_mqa = hq != hk
418
+ off_h_k = off_h_q % hk if is_mqa else off_h_q
419
+ n_extra_tokens = 0
420
+ if seqlen_k < BLOCK_N:
421
+ n_extra_tokens = BLOCK_N - seqlen_k
422
+ elif seqlen_k % BLOCK_N:
423
+ n_extra_tokens = seqlen_k % BLOCK_N
424
+ padded_head = ACTUAL_BLOCK_DMODEL != BLOCK_DMODEL
425
+
426
+ # Compute pointers for all the tensors used in this kernel.
427
+ q_offset = (off_z * stride_qz + off_h_q * stride_qh +
428
+ cu_seqlens_q_start * stride_qm)
429
+ Q_block_ptr = tl.make_block_ptr(
430
+ base=Q + q_offset,
431
+ shape=(seqlen_q, ACTUAL_BLOCK_DMODEL),
432
+ strides=(stride_qm, stride_qk),
433
+ offsets=(start_m * BLOCK_M, 0),
434
+ block_shape=(BLOCK_M, BLOCK_DMODEL),
435
+ order=(1, 0),
436
+ )
437
+ k_offset = (off_z * stride_kz + off_h_k * stride_kh +
438
+ cu_seqlens_k_start * stride_kn)
439
+ K_block_ptr = tl.make_block_ptr(
440
+ base=K + k_offset,
441
+ shape=(ACTUAL_BLOCK_DMODEL, seqlen_k),
442
+ strides=(stride_kk, stride_kn),
443
+ offsets=(0, 0),
444
+ block_shape=(BLOCK_DMODEL, BLOCK_N),
445
+ order=(0, 1),
446
+ )
447
+ v_offset = (off_z * stride_vz + off_h_k * stride_vh +
448
+ cu_seqlens_k_start * stride_vk)
449
+ V_block_ptr = tl.make_block_ptr(
450
+ base=V + v_offset,
451
+ shape=(seqlen_k, ACTUAL_BLOCK_DMODEL),
452
+ strides=(stride_vk, stride_vn),
453
+ offsets=(0, 0),
454
+ block_shape=(BLOCK_N, BLOCK_DMODEL),
455
+ order=(1, 0),
456
+ )
457
+ if BIAS_TYPE != 0:
458
+ bias_ptr = tl.make_block_ptr(
459
+ base=bias + off_h_q * stride_bh,
460
+ shape=(seqlen_q, seqlen_k),
461
+ strides=(stride_bm, stride_bn),
462
+ offsets=(start_m * BLOCK_M, 0),
463
+ block_shape=(BLOCK_M, BLOCK_N),
464
+ order=(1, 0),
465
+ )
466
+ else:
467
+ bias_ptr = None
468
+ if ENABLE_DROPOUT:
469
+ batch_philox_offset = philox_offset_base \
470
+ + (off_z * hq + off_h_q) \
471
+ * seqlen_q * seqlen_k
472
+ else:
473
+ batch_philox_offset = 0
474
+ # We can ask to return the dropout mask without actually doing any dropout.
475
+ # In this case, we return an invalid pointer so indicate the mask is not i
476
+ # valid.
477
+ # TODO: Fix encoded softmax. It currently uses just h_q in the base offset.
478
+ if RETURN_ENCODED_SOFTMAX:
479
+ encoded_softmax_block_ptr = tl.make_block_ptr(
480
+ base=encoded_softmax + off_h_q * seqlen_q * seqlen_k,
481
+ shape=(seqlen_q, seqlen_k),
482
+ strides=(seqlen_k, 1),
483
+ offsets=(start_m * BLOCK_M, 0),
484
+ block_shape=(BLOCK_M, BLOCK_N),
485
+ order=(1, 0),
486
+ )
487
+ else:
488
+ encoded_softmax_block_ptr = 0
489
+ # initialize pointer to m and l
490
+ m_i = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32)
491
+ l_i = tl.full([BLOCK_M], 1.0, dtype=tl.float32)
492
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
493
+ # scale sm_scale by log_2(e) and use 2^x in the loop as we do not
494
+ # have native e^x support in HW.
495
+ qk_scale = sm_scale * 1.44269504089
496
+ # Q is loaded once at the beginning and shared by all N blocks.
497
+ q = load_fn(Q_block_ptr, True, padded_head, "zero")
498
+ q = (q * qk_scale).to(Q_block_ptr.type.element_ty)
499
+
500
+ # Here we compute how many full and masked blocks we have.
501
+ padded_block_k = n_extra_tokens != 0
502
+ is_modulo_mn = not padded_block_k and (seqlen_q % BLOCK_M == 0)
503
+ if IS_CAUSAL:
504
+ # There are always at least BLOCK_M // BLOCK_N masked blocks.
505
+ # Additionally there might be one more due to dissimilar seqlens.
506
+ masked_blocks = BLOCK_M // BLOCK_N + (not is_modulo_mn)
507
+ else:
508
+ # Padding on Q does not need to be masked in the FA loop.
509
+ masked_blocks = padded_block_k
510
+ # if IS_CAUSAL, not is_modulo_mn does not always result in an additional
511
+ # block. In this case we might exceed n_blocks so pick the min.
512
+ masked_blocks = min(masked_blocks, n_blocks)
513
+ n_full_blocks = n_blocks - masked_blocks
514
+ block_min = 0
515
+ block_max = n_blocks * BLOCK_N
516
+ # Compute for full blocks. Here we set causal to false regardless of its
517
+ # value because there is no masking. Similarly we do not need padding.
518
+ if n_full_blocks > 0:
519
+ block_max = (n_blocks - masked_blocks) * BLOCK_N
520
+ acc, l_i, m_i = _attn_fwd_inner(
521
+ acc,
522
+ l_i,
523
+ m_i,
524
+ q,
525
+ K_block_ptr,
526
+ V_block_ptr,
527
+ start_m,
528
+ seqlen_k,
529
+ dropout_p,
530
+ philox_seed,
531
+ batch_philox_offset,
532
+ encoded_softmax_block_ptr,
533
+ # _, _, offs_n_causal, masked_blocks, n_extra_tokens, _
534
+ block_min,
535
+ block_max,
536
+ 0,
537
+ 0,
538
+ 0,
539
+ bias_ptr,
540
+ # IS_CAUSAL, ....
541
+ False,
542
+ BLOCK_M,
543
+ BLOCK_DMODEL,
544
+ BLOCK_N,
545
+ offs_m,
546
+ offs_n,
547
+ # _, MASK_STEPS, ...
548
+ PRE_LOAD_V,
549
+ False,
550
+ ENABLE_DROPOUT,
551
+ RETURN_ENCODED_SOFTMAX,
552
+ padded_head,
553
+ )
554
+ block_min = block_max
555
+ block_max = n_blocks * BLOCK_N
556
+
557
+ tl.debug_barrier()
558
+ # Remaining blocks, if any, are full / not masked.
559
+ if masked_blocks > 0:
560
+ offs_n_causal = offs_n + (seqlen_q - seqlen_k) if IS_CAUSAL else 0
561
+ K_block_ptr = tl.advance(K_block_ptr, (0, n_full_blocks * BLOCK_N))
562
+ V_block_ptr = tl.advance(V_block_ptr, (n_full_blocks * BLOCK_N, 0))
563
+ if bias_ptr is not None:
564
+ bias_ptr = tl.advance(bias_ptr, (0, n_full_blocks * BLOCK_N))
565
+ if RETURN_ENCODED_SOFTMAX:
566
+ encoded_softmax_block_ptr = tl.advance(encoded_softmax_block_ptr,
567
+ (0, n_full_blocks))
568
+ acc, l_i, m_i = _attn_fwd_inner(
569
+ acc,
570
+ l_i,
571
+ m_i,
572
+ q,
573
+ K_block_ptr,
574
+ V_block_ptr,
575
+ start_m,
576
+ seqlen_k,
577
+ dropout_p,
578
+ philox_seed,
579
+ batch_philox_offset,
580
+ encoded_softmax_block_ptr,
581
+ block_min,
582
+ block_max,
583
+ offs_n_causal,
584
+ masked_blocks,
585
+ n_extra_tokens,
586
+ bias_ptr,
587
+ IS_CAUSAL,
588
+ BLOCK_M,
589
+ BLOCK_DMODEL,
590
+ BLOCK_N,
591
+ offs_m,
592
+ offs_n,
593
+ # _, MASK_STEPS, ...
594
+ PRE_LOAD_V,
595
+ True,
596
+ ENABLE_DROPOUT,
597
+ RETURN_ENCODED_SOFTMAX,
598
+ padded_head,
599
+ )
600
+ # epilogue
601
+ acc = acc / l_i[:, None]
602
+ if ENABLE_DROPOUT:
603
+ acc = acc / (1 - dropout_p)
604
+ # If seqlen_q > seqlen_k but the delta is not a multiple of BLOCK_M,
605
+ # then we have one block with a row of all NaNs which come from computing
606
+ # softmax over a row of all -infs (-inf - inf = NaN). We check for that here
607
+ # and store 0s where there are NaNs as these rows should've been zeroed out.
608
+ end_m_idx = (start_m + 1) * BLOCK_M
609
+ start_m_idx = start_m * BLOCK_M
610
+ causal_start_idx = seqlen_q - seqlen_k
611
+ acc = acc.to(Out.type.element_ty)
612
+ if IS_CAUSAL: # noqa: SIM102
613
+ if causal_start_idx > start_m_idx and causal_start_idx < end_m_idx:
614
+ out_mask_boundary = tl.full((BLOCK_DMODEL, ),
615
+ causal_start_idx,
616
+ dtype=tl.int32)
617
+ mask_m_offsets = start_m_idx + tl.arange(0, BLOCK_M)
618
+ out_ptrs_mask = (mask_m_offsets[:, None] >=
619
+ out_mask_boundary[None, :])
620
+ z = 0.0
621
+ acc = tl.where(out_ptrs_mask, acc, z.to(acc.type.element_ty))
622
+ # write back LSE
623
+ # l_ptrs = L + off_z * hq * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q + offs_m
624
+ # If seqlen_q not multiple of BLOCK_M, we need to mask out the last
625
+ # few rows. This is only true for the last M block. For others,
626
+ # overflow_size will be -ve
627
+ # overflow_size = end_m_idx - seqlen_q
628
+ # if overflow_size > 0:
629
+ # boundary = tl.full((BLOCK_M,), BLOCK_M - overflow_size, dtype=tl.int32)
630
+ # # This is a > check because mask being 0 blocks the store.
631
+ # l_ptrs_mask = boundary > tl.arange(0, BLOCK_M)
632
+ # tl.store(l_ptrs, m_i + tl.math.log2(l_i), mask=l_ptrs_mask)
633
+ # else:
634
+ # tl.store(l_ptrs, m_i + tl.math.log2(l_i))
635
+
636
+ # write back O
637
+ o_offset = (off_z * stride_oz + cu_seqlens_q_start * stride_om +
638
+ off_h_q * stride_oh)
639
+ O_block_ptr = tl.make_block_ptr(
640
+ base=Out + o_offset,
641
+ shape=(seqlen_q, ACTUAL_BLOCK_DMODEL),
642
+ strides=(stride_om, stride_on),
643
+ offsets=(start_m * BLOCK_M, 0),
644
+ block_shape=(BLOCK_M, BLOCK_DMODEL),
645
+ order=(1, 0),
646
+ )
647
+ # Need boundary check on this to make sure the padding from the
648
+ # Q and KV tensors in both dims are not part of what we store back.
649
+ # TODO: Do the boundary check optionally.
650
+ tl.store(O_block_ptr, acc, boundary_check=(0, 1))
651
+
652
+
653
+def check_args(
654
+ q,
655
+ k,
656
+ v,
657
+ o,
658
+ varlen=True,
659
+ max_seqlens=None,
660
+ cu_seqlens_q=None,
661
+ cu_seqlens_k=None,
662
+):
663
+ assert q.dim() == k.dim() and q.dim() == v.dim()
664
+ if varlen:
665
+ assert q.dim() == 3
666
+ total_q, nheads_q, head_size = q.shape
667
+ total_k, nheads_k, _ = k.shape
668
+ assert cu_seqlens_q is not None
669
+ assert cu_seqlens_k is not None
670
+ assert len(cu_seqlens_q) == len(cu_seqlens_k)
671
+ else:
672
+ assert q.dim() == 4
673
+ batch, nheads_q, seqlen_q, head_size = q.shape
674
+ _, nheads_k, seqlen_k, _ = k.shape
675
+ assert max_seqlens > 0
676
+ assert k.shape == v.shape
677
+ assert q.shape[-1] == k.shape[-1] and q.shape[-1] == v.shape[-1]
678
+ # TODO: Change assert if we support qkl f8 and v f16
679
+ assert q.dtype == k.dtype and q.dtype == v.dtype
680
+ # TODO: Fix assert to check head size <=256 once supported
681
+ assert head_size <= 128
682
+ assert o.shape == q.shape
683
+ assert (nheads_q % nheads_k) == 0
684
+
685
+
686
+class _attention(torch.autograd.Function):
687
+
688
+ @staticmethod
689
+ def forward(
690
+ ctx,
691
+ q,
692
+ k,
693
+ v,
694
+ o,
695
+ cu_seqlens_q,
696
+ cu_seqlens_k,
697
+ max_seqlens_q,
698
+ max_seqlens_k,
699
+ causal=False,
700
+ sm_scale=1.0,
701
+ bias=None,
702
+ ):
703
+ if o is None:
704
+ o = torch.empty_like(q, dtype=v.dtype)
705
+
706
+ check_args(
707
+ q,
708
+ k,
709
+ v,
710
+ o,
711
+ varlen=True,
712
+ cu_seqlens_q=cu_seqlens_q,
713
+ cu_seqlens_k=cu_seqlens_k,
714
+ )
715
+ if True: # varlen
716
+ total_q, nheads_q, head_size = q.shape
717
+ total_k, nheads_k, _ = k.shape
718
+ batch = len(cu_seqlens_q) - 1
719
+ q_strides = (0, q.stride(1), q.stride(0), q.stride(2))
720
+ k_strides = (0, k.stride(1), k.stride(0), k.stride(2))
721
+ v_strides = (0, v.stride(1), v.stride(0), v.stride(2))
722
+ o_strides = (0, o.stride(1), o.stride(0), o.stride(2))
723
+ else:
724
+ batch, seqlen_q, nheads_q, head_size = q.shape
725
+ _, seqlen_k, nheads_k, _ = k.shape
726
+ q_strides = (q.stride(0), q.stride(2), q.stride(1), q.stride(3))
727
+ k_strides = (k.stride(0), k.stride(2), k.stride(1), k.stride(3))
728
+ v_strides = (v.stride(0), v.stride(2), v.stride(1), v.stride(3))
729
+ o_strides = (o.stride(0), o.stride(2), o.stride(1), o.stride(3))
730
+
731
+ # Get closest power of 2 over or equal to 32.
732
+ unpadded_head_dims = {32, 64, 128}
733
+ if head_size not in unpadded_head_dims:
734
+ padded_d_model = None
735
+ for i in unpadded_head_dims:
736
+ if i > head_size:
737
+ padded_d_model = i
738
+ break
739
+ assert padded_d_model is not None
740
+ else:
741
+ padded_d_model = head_size
742
+
743
+ grid = lambda META: (
744
+ triton.cdiv(max_seqlens_q, META["BLOCK_M"]),
745
+ nheads_q,
746
+ batch,
747
+ )
748
+
749
+ encoded_softmax = None
750
+
751
+ # Seed the RNG so we get reproducible results for testing.
752
+ philox_seed = 0x1BF52
753
+ philox_offset = 0x1D4B42
754
+
755
+ if bias is not None:
756
+ bias_strides = (
757
+ bias.stride(0),
758
+ bias.stride(1),
759
+ bias.stride(2),
760
+ bias.stride(3),
761
+ )
762
+ else:
763
+ bias_strides = (0, 0, 0, 0)
764
+
765
+ attn_fwd[grid](
766
+ q,
767
+ k,
768
+ v,
769
+ bias,
770
+ sm_scale,
771
+ None,
772
+ o,
773
+ *q_strides,
774
+ *k_strides,
775
+ *v_strides,
776
+ *o_strides,
777
+ *bias_strides,
778
+ cu_seqlens_q,
779
+ cu_seqlens_k,
780
+ dropout_p=0.0,
781
+ philox_seed=philox_seed,
782
+ philox_offset_base=philox_offset,
783
+ encoded_softmax=encoded_softmax,
784
+ hq=nheads_q,
785
+ hk=nheads_k,
786
+ ACTUAL_BLOCK_DMODEL=head_size,
787
+ MAX_SEQLENS_Q=max_seqlens_q,
788
+ MAX_SEQLENS_K=max_seqlens_k,
789
+ IS_CAUSAL=causal,
790
+ VARLEN=True,
791
+ BLOCK_DMODEL=padded_d_model,
792
+ BIAS_TYPE=0 if bias is None else 1,
793
+ ENABLE_DROPOUT=False,
794
+ RETURN_ENCODED_SOFTMAX=False,
795
+ )
796
+
797
+ ctx.grid = grid
798
+ ctx.sm_scale = sm_scale
799
+ ctx.BLOCK_DMODEL = head_size
800
+ ctx.causal = causal
801
+ ctx.dropout_p = 0.0
802
+ ctx.philox_seed = philox_seed
803
+ ctx.philox_offset = philox_offset
804
+ ctx.encoded_softmax = encoded_softmax
805
+ ctx.return_encoded_softmax = False
806
+ return o, encoded_softmax
807
+
808
+
809
+triton_attention = _attention.apply
vllm/attention/selector.py
+40
-17
@@ -1,3 +1,4 @@
1
+import enum
2
from functools import lru_cache
3
from typing import Type
4
@@ -10,46 +11,68 @@ from vllm.utils import is_cpu, is_hip
11
logger = init_logger(__name__)
12
13
14
+class _Backend(enum.Enum):
15
+ FLASH_ATTN = enum.auto()
16
+ XFORMERS = enum.auto()
17
+ ROCM_FLASH = enum.auto()
18
+ TORCH_SDPA = enum.auto()
19
+
20
+
21
@lru_cache(maxsize=None)
22
def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
15
- if _can_use_flash_attn(dtype):
23
+ backend = _which_attn_to_use(dtype)
24
+ if backend == _Backend.FLASH_ATTN:
25
logger.info("Using FlashAttention backend.")
26
from vllm.attention.backends.flash_attn import ( # noqa: F401
27
FlashAttentionBackend)
28
return FlashAttentionBackend
20
- elif is_cpu():
21
- logger.info("Using Torch SDPA backend.")
22
- from vllm.attention.backends.torch_sdpa import TorchSDPABackend
23
- return TorchSDPABackend
24
- else:
29
+ elif backend == _Backend.XFORMERS:
30
logger.info("Using XFormers backend.")
31
from vllm.attention.backends.xformers import ( # noqa: F401
32
XFormersBackend)
33
return XFormersBackend
34
+ elif backend == _Backend.ROCM_FLASH:
35
+ logger.info("Using ROCmFlashAttention backend.")
36
+ from vllm.attention.backends.rocm_flash_attn import ( # noqa: F401
37
+ ROCmFlashAttentionBackend)
38
+ return ROCmFlashAttentionBackend
39
+ elif backend == _Backend.TORCH_SDPA:
40
+ logger.info("Using Torch SDPA backend.")
41
+ from vllm.attention.backends.torch_sdpa import TorchSDPABackend
42
+ return TorchSDPABackend
43
+ else:
44
+ raise ValueError("Invalid attention backend.")
45
46
31
-def _can_use_flash_attn(dtype: torch.dtype) -> bool:
47
+def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
48
+ """Returns which flash attention backend to use."""
49
+ if is_cpu():
50
+ return _Backend.TORCH_SDPA
51
+
52
if is_hip():
53
# AMD GPUs.
34
- logger.info("Cannot use FlashAttention backend for AMD GPUs.")
35
- return False
36
- if is_cpu():
37
- return False
54
+ if torch.cuda.get_device_capability()[0] != 9:
55
+ # not Instinct series GPUs.
56
+ logger.info("flash_atten is not supported on NAVI GPUs.")
57
+ return _Backend.ROCM_FLASH
58
+
59
+ # NVIDIA GPUs.
60
if torch.cuda.get_device_capability()[0] < 8:
61
# Volta and Turing NVIDIA GPUs.
62
logger.info("Cannot use FlashAttention backend for Volta and Turing "
63
"GPUs.")
42
- return False
64
+ return _Backend.XFORMERS
65
+
66
if dtype not in (torch.float16, torch.bfloat16):
67
logger.info("Cannot use FlashAttention backend for dtype other than "
68
"torch.float16 or torch.bfloat16.")
46
- return False
69
+ return _Backend.XFORMERS
70
71
try:
72
import flash_attn # noqa: F401
73
except ImportError:
74
logger.info(
52
- "Cannot use FlashAttention because the package is not found. "
53
- "Please install it for better performance.")
54
- return False
55
- return True
75
+ "Cannot use FlashAttention backend because the flash_attn package "
76
+ "is not found. Please install it for better performance.")
77
+ return _Backend.XFORMERS
78
+ return _Backend.FLASH_ATTN