@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c0b04515f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[ROCm][Hardware][AMD] Use Triton Kernel for default FA on ROCm (#3643)

Co-authored-by: jpvillam <jpvillam@amd.com> Co-authored-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Juan Villamizar committed Apr 9, 2024 at 17:10 UTC 6c0b04515fee7b402a6febde1467581825bb2164
5 files changed +1213 -93
Dockerfile.rocm
+14
@@ -23,6 +23,9 @@ RUN echo "FA_BRANCH is $FA_BRANCH"
23 # In that case, we need to use the python reference attention implementation in vllm
24 ARG BUILD_FA="1"
25
26 +# whether to build triton on rocm
27 +ARG BUILD_TRITON="1"
28 +
29 # Install some basic utilities
30 RUN apt-get update && apt-get install python3 python3-pip -y
31
@@ -75,6 +78,17 @@ RUN if [ "$BUILD_FA" = "1" ]; then \
78 RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" ]; then \
79 rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
80
81 +# build triton
82 +RUN if [ "$BUILD_TRITON" = "1" ]; then \
83 + mkdir -p libs \
84 + && cd libs \
85 + && pip uninstall -y triton \
86 + && git clone https://github.com/ROCm/triton.git \
87 + && cd triton/python \
88 + && pip3 install . \
89 + && cd ../..; \
90 + fi
91 +
92 COPY ./ /app/vllm
93
94 RUN python3 -m pip install --upgrade pip
vllm/attention/backends/rocm_flash_attn.py new
+348
@@ -0,0 +1,348 @@
1 +"""Attention layer ROCm GPUs."""
2 +import os
3 +from dataclasses import dataclass
4 +from typing import Dict, List, Optional, Tuple, Type
5 +
6 +import torch
7 +
8 +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 + AttentionMetadata)
10 +from vllm.attention.ops.paged_attn import (PagedAttention,
11 + PagedAttentionMetadata)
12 +from vllm.logger import init_logger
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +class ROCmFlashAttentionBackend(AttentionBackend):
18 +
19 + @staticmethod
20 + def get_impl_cls() -> Type["ROCmFlashAttentionImpl"]:
21 + return ROCmFlashAttentionImpl
22 +
23 + @staticmethod
24 + def make_metadata(*args, **kwargs) -> "ROCmFlashAttentionMetadata":
25 + return ROCmFlashAttentionMetadata(*args, **kwargs)
26 +
27 + @staticmethod
28 + def get_kv_cache_shape(
29 + num_blocks: int,
30 + block_size: int,
31 + num_kv_heads: int,
32 + head_size: int,
33 + ) -> Tuple[int, ...]:
34 + return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
35 + num_kv_heads, head_size)
36 +
37 + @staticmethod
38 + def swap_blocks(
39 + src_kv_cache: torch.Tensor,
40 + dst_kv_cache: torch.Tensor,
41 + src_to_dst: Dict[int, int],
42 + ) -> None:
43 + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
44 +
45 + @staticmethod
46 + def copy_blocks(
47 + kv_caches: List[torch.Tensor],
48 + src_to_dists: Dict[int, List[int]],
49 + ) -> None:
50 + PagedAttention.copy_blocks(kv_caches, src_to_dists)
51 +
52 +
53 +@dataclass
54 +class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
55 + """Metadata for FlashAttentionBackend.
56 +
57 + NOTE: Any python object stored here is not updated when it is
58 + cuda-graph replayed. If you have values that need to be changed
59 + dynamically, it should be stored in tensor. The tensor has to be
60 + updated from `CUDAGraphRunner.forward` API.
61 + """
62 + # Currently, input sequences can only contain all prompts
63 + # or all decoding. True if all sequences are prompts.
64 + is_prompt: bool
65 + # (batch_size,). The prompt length per sequence. None if it is a decoding.
66 + prompt_lens: Optional[List[int]]
67 + # prompt_lens stored as a tensor.
68 + prompt_lens_tensor: Optional[torch.Tensor]
69 + # The number of prompt tokens. Doesn't include padding.
70 + num_prompt_tokens: int
71 + # The number of generation tokens. Doesn't include padding.
72 + num_generation_tokens: int
73 +
74 + # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75 + # |---------- N-1 iteration --------|
76 + # |---------------- N iteration ---------------------|
77 + # |- tokenA -|......................|-- newTokens ---|
78 + # |---------- context_len ----------|
79 + # |-------------------- seqlen ----------------------|
80 + # |- subquery_len -|
81 +
82 + # WARNING(sang): context_len has different definition depending on if it is
83 + # prefill vs decoding. When it is prefill, it doesn't include new tokens.
84 + # When it is for decoding, it includes a new token.
85 +
86 + # Maximum subquery length in the batch.
87 + max_subquery_len: Optional[int]
88 + # Maximum prompt length in the batch.
89 + max_prompt_len: Optional[int]
90 + # (batch_size + 1,). The cumulative subquery lengths of the sequences in
91 + # the batch, used to index into subquery. E.g., if the subquery length
92 + # is [4, 6], it is [0, 4, 10].
93 + subquery_start_loc: Optional[torch.Tensor]
94 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
95 + # the batch, used to index into sequence. E.g., if the sequence length is
96 + # [4, 6], it is [0, 4, 10].
97 + seq_start_loc: Optional[torch.Tensor]
98 +
99 + # Whether or not if cuda graph is enabled.
100 + # Cuda-graph is currently enabled for decoding only.
101 + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
102 + use_cuda_graph: bool
103 +
104 +
105 +class ROCmFlashAttentionImpl(AttentionImpl):
106 + """
107 + If the input tensors contain prompt tokens, the layout is as follows:
108 + |<--------------- num_prompt_tokens -------------->|
109 + |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
110 +
111 + Otherwise, the layout is as follows:
112 + |<------------------ num_generation_tokens (M) ----------------->|
113 + |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
114 +
115 + Generation tokens can contain padding when cuda-graph is used.
116 + Currently, prompt tokens don't contain any padding.
117 +
118 + The prompts might have different lengths, while the generation tokens
119 + always have length 1.
120 + """
121 +
122 + def __init__(
123 + self,
124 + num_heads: int,
125 + head_size: int,
126 + scale: float,
127 + num_kv_heads: Optional[int] = None,
128 + alibi_slopes: Optional[List[float]] = None,
129 + sliding_window: Optional[int] = None,
130 + ) -> None:
131 + self.num_heads = num_heads
132 + self.head_size = head_size
133 + self.scale = float(scale)
134 + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
135 + self.sliding_window = ((sliding_window, sliding_window)
136 + if sliding_window is not None else (-1, -1))
137 + if alibi_slopes is not None:
138 + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
139 + self.alibi_slopes = alibi_slopes
140 +
141 + assert self.num_heads % self.num_kv_heads == 0
142 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
143 +
144 + suppored_head_sizes = PagedAttention.get_supported_head_sizes()
145 + if head_size not in suppored_head_sizes:
146 + raise ValueError(
147 + f"Head size {head_size} is not supported by PagedAttention. "
148 + f"Supported head sizes are: {suppored_head_sizes}.")
149 +
150 + self.use_naive_attn = torch.cuda.get_device_capability()[0] != 9
151 + # NOTE: Allow for switching between Triton and CK. Defaulting to triton.
152 + self.use_triton_flash_attn = (os.environ.get(
153 + "VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in ("true", "1"))
154 + if self.use_naive_attn:
155 + # AMD Radeon 7900 series (gfx1100) currently does not support
156 + # xFormers nor FlashAttention. As a temporary workaround, we use
157 + # naive PyTorch implementation of attention.
158 + self.attn_fuc = _naive_attention()
159 + logger.debug("Using naive attention in ROCmBackend")
160 + elif self.use_triton_flash_attn:
161 + from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
162 + triton_attention)
163 + self.attn_func = triton_attention
164 + logger.debug("Using Triton FA in ROCmBackend")
165 + else:
166 + from flash_attn import flash_attn_varlen_func # noqa: F401
167 + self.attn_func = flash_attn_varlen_func
168 + logger.debug("Using CK FA in ROCmBackend")
169 +
170 + def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
171 + """torch.repeat_interleave(x, dim=1, repeats=n_rep)"""
172 + tokens, n_kv_heads, head_dim = x.shape
173 + return (x[:, :,
174 + None, :].expand(tokens, n_kv_heads, n_rep,
175 + head_dim).reshape(tokens, n_kv_heads * n_rep,
176 + head_dim))
177 +
178 + def forward(
179 + self,
180 + query: torch.Tensor,
181 + key: torch.Tensor,
182 + value: torch.Tensor,
183 + kv_cache: torch.Tensor,
184 + attn_metadata: ROCmFlashAttentionMetadata,
185 + kv_scale: float = 1.0,
186 + ) -> torch.Tensor:
187 + """Forward pass with FlashAttention and PagedAttention.
188 +
189 + Args:
190 + query: shape = [num_tokens, num_heads * head_size]
191 + key: shape = [num_tokens, num_kv_heads * head_size]
192 + value: shape = [num_tokens, num_kv_heads * head_size]
193 + kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
194 + attn_metadata: Metadata for attention.
195 + Returns:
196 + shape = [num_tokens, num_heads * head_size]
197 + """
198 + num_tokens, hidden_size = query.shape
199 + # Reshape the query, key, and value tensors.
200 + query = query.view(-1, self.num_heads, self.head_size)
201 + key = key.view(-1, self.num_kv_heads, self.head_size)
202 + value = value.view(-1, self.num_kv_heads, self.head_size)
203 +
204 + if kv_cache is not None:
205 + key_cache, value_cache = PagedAttention.split_kv_cache(
206 + kv_cache, self.num_kv_heads, self.head_size)
207 +
208 + # Reshape the input keys and values and store them in the cache.
209 + # If kv_cache is not provided, the new key and value tensors are
210 + # not cached. This happens during the initial memory profiling run.
211 + PagedAttention.write_to_paged_cache(
212 + key,
213 + value,
214 + key_cache,
215 + value_cache,
216 + attn_metadata.slot_mapping,
217 + attn_metadata.kv_cache_dtype,
218 + kv_scale,
219 + )
220 +
221 + if attn_metadata.is_prompt:
222 + # Prompt run.
223 + if kv_cache is None or attn_metadata.block_tables.numel() == 0:
224 + # triton attention
225 + # When block_tables are not filled, it means q and k are the
226 + # prompt, and they have the same length.
227 + if self.use_naive_attn or self.use_triton_flash_attn:
228 + if self.num_kv_heads != self.num_heads:
229 + # Interleave for MQA workaround.
230 + key = self.repeat_kv(key, self.num_queries_per_kv)
231 + value = self.repeat_kv(value, self.num_queries_per_kv)
232 + if self.use_naive_attn:
233 + output = self.attn_fuc(
234 + query,
235 + key,
236 + value,
237 + attn_metadata.prompt_lens,
238 + self.scale,
239 + )
240 + else:
241 + output, _ = self.attn_func(
242 + query,
243 + key,
244 + value,
245 + None,
246 + attn_metadata.seq_start_loc,
247 + attn_metadata.seq_start_loc,
248 + attn_metadata.max_prompt_len,
249 + attn_metadata.max_prompt_len,
250 + True,
251 + self.scale,
252 + )
253 + else:
254 + output = self.attn_func(
255 + q=query,
256 + k=key,
257 + v=value,
258 + cu_seqlens_q=attn_metadata.seq_start_loc,
259 + cu_seqlens_k=attn_metadata.seq_start_loc,
260 + max_seqlen_q=attn_metadata.max_prompt_len,
261 + max_seqlen_k=attn_metadata.max_prompt_len,
262 + softmax_scale=self.scale,
263 + causal=True,
264 + )
265 +
266 + else:
267 + # prefix-enabled attention
268 + output = PagedAttention.forward_prefix(
269 + query,
270 + key,
271 + value,
272 + key_cache,
273 + value_cache,
274 + attn_metadata.block_tables,
275 + attn_metadata.subquery_start_loc,
276 + attn_metadata.prompt_lens_tensor,
277 + attn_metadata.context_lens,
278 + attn_metadata.max_subquery_len,
279 + self.alibi_slopes,
280 + )
281 + else:
282 + # Decoding run.
283 + output = PagedAttention.forward_decode(
284 + query,
285 + key_cache,
286 + value_cache,
287 + attn_metadata.block_tables,
288 + attn_metadata.context_lens,
289 + attn_metadata.max_context_len,
290 + attn_metadata.kv_cache_dtype,
291 + self.num_kv_heads,
292 + self.scale,
293 + self.alibi_slopes,
294 + kv_scale,
295 + )
296 +
297 + # Reshape the output tensor.
298 + return output.view(num_tokens, hidden_size)
299 +
300 +
301 +def _naive_attention(
302 + query: torch.Tensor,
303 + key: torch.Tensor,
304 + value: torch.Tensor,
305 + prompt_lens: List[int],
306 + scale: float,
307 +) -> torch.Tensor:
308 + num_tokens = query.shape[0]
309 + output = torch.empty_like(query)
310 + start = 0
311 + for _, prompt_len in enumerate(prompt_lens):
312 + end = start + prompt_len
313 + out = _naive_masked_attention(
314 + query[None, start:end],
315 + key[None, start:end],
316 + value[None, start:end],
317 + scale,
318 + )
319 + # TODO(woosuk): Unnecessary copy. Optimize.
320 + output[start:end].copy_(out)
321 + start += prompt_len
322 +
323 + # Using view got RuntimeError: view size is not compatible
324 + # with input tensor's size and stride (at least one
325 + # dimension spans across two contiguous subspaces).
326 + # Use reshape instead.
327 + return output.reshape(num_tokens, -1)
328 +
329 +
330 +def _naive_masked_attention(
331 + query: torch.Tensor,
332 + key: torch.Tensor,
333 + value: torch.Tensor,
334 + scale: float,
335 +) -> torch.Tensor:
336 + seq_len, _, _ = query.shape
337 + attn_mask = torch.triu(torch.ones(seq_len,
338 + seq_len,
339 + dtype=query.dtype,
340 + device=query.device),
341 + diagonal=1)
342 + attn_mask = attn_mask * torch.finfo(query.dtype).min
343 +
344 + attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
345 + attn_weights = attn_weights + attn_mask.float()
346 + attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
347 + out = torch.einsum("hqk,khd->qhd", attn_weights, value)
348 + return out
vllm/attention/backends/xformers.py
+2 -76
@@ -1,5 +1,4 @@
1 """Attention layer with xFormers and PagedAttention."""
2 -import importlib
2 from dataclasses import dataclass
3 from typing import Dict, List, Optional, Tuple, Type
4
@@ -14,7 +13,6 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 from vllm.attention.ops.paged_attn import (PagedAttention,
14 PagedAttentionMetadata)
15 from vllm.logger import init_logger
17 -from vllm.utils import is_hip
16
17 logger = init_logger(__name__)
18
@@ -166,11 +164,6 @@ class XFormersImpl(AttentionImpl):
164 f"Head size {head_size} is not supported by PagedAttention. "
165 f"Supported head sizes are: {suppored_head_sizes}.")
166
169 - # AMD Radeon 7900 series (gfx1100) currently does not support xFormers
170 - # nor FlashAttention. As a temporary workaround, we use naive PyTorch
171 - # implementation of attention.
172 - self.use_naive_attention = _check_use_naive_attention()
173 -
167 def forward(
168 self,
169 query: torch.Tensor,
@@ -233,30 +226,6 @@ class XFormersImpl(AttentionImpl):
226 self.num_queries_per_kv,
227 value.shape[-1])
228
236 - if self.use_naive_attention:
237 - output = torch.empty_like(query)
238 - start = 0
239 - for _, prompt_len in enumerate(attn_metadata.prompt_lens):
240 - end = start + prompt_len
241 - out = _naive_masked_attention(
242 - query[None, start:end],
243 - key[None, start:end],
244 - value[None, start:end],
245 - self.num_heads,
246 - self.num_kv_heads,
247 - self.head_size,
248 - self.scale,
249 - )
250 - # TODO(woosuk): Unnecessary copy. Optimize.
251 - output[start:end].copy_(out)
252 - start += prompt_len
253 -
254 - # Using view got RuntimeError: view size is not compatible
255 - # with input tensor's size and stride (at least one
256 - # dimension spans across two contiguous subspaces).
257 - # Use reshape instead.
258 - return output.reshape(num_tokens, hidden_size)
259 -
229 output = self._run_memory_efficient_xformers_forward(
230 query, key, value, attn_metadata)
231 else:
@@ -329,8 +298,6 @@ class XFormersImpl(AttentionImpl):
298 self.alibi_slopes, self.num_kv_heads, query.dtype,
299 attn_metadata.prompt_lens)
300
332 - op = xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if (
333 - is_hip()) else None
301 # No alibi slopes.
302 # TODO(woosuk): Too many view operations. Let's try to reduce
303 # them in the future for code readability.
@@ -344,8 +311,7 @@ class XFormersImpl(AttentionImpl):
311 value,
312 attn_bias=attn_metadata.attn_bias[0],
313 p=0.0,
347 - scale=self.scale,
348 - op=op)
314 + scale=self.scale)
315
316 return out.view_as(query)
317
@@ -363,8 +329,7 @@ class XFormersImpl(AttentionImpl):
329 value[None, start:end],
330 attn_bias=attn_metadata.attn_bias[i],
331 p=0.0,
366 - scale=self.scale,
367 - op=op)
332 + scale=self.scale)
333 # TODO(woosuk): Unnecessary copy. Optimize.
334 output[start:end].copy_(out.squeeze(0))
335 start += prompt_len
@@ -405,42 +370,3 @@ def _make_alibi_bias(
370 attn_biases.append(LowerTriangularMaskWithTensorBias(bias))
371
372 return attn_biases
408 -
409 -
410 -def _check_use_naive_attention() -> bool:
411 - if not is_hip():
412 - return False
413 - # For ROCm, check whether flash attention is installed or not.
414 - use_naive_attention = importlib.util.find_spec("flash_attn") is None
415 - if use_naive_attention:
416 - logger.warning("flash_attn is not installed. Using naive attention. "
417 - "This will take significantly more GPU memory.")
418 - return True
419 - return False
420 -
421 -
422 -def _naive_masked_attention(
423 - query: torch.Tensor,
424 - key: torch.Tensor,
425 - value: torch.Tensor,
426 - num_heads: int,
427 - num_kv_heads: int,
428 - head_size: int,
429 - scale: float,
430 -) -> torch.Tensor:
431 - query = query.view(-1, num_heads, head_size)
432 - key = key.view(-1, num_kv_heads, head_size)
433 - value = value.view(-1, num_kv_heads, head_size)
434 - seq_len, _, _ = query.shape
435 - attn_mask = torch.triu(torch.ones(seq_len,
436 - seq_len,
437 - dtype=query.dtype,
438 - device=query.device),
439 - diagonal=1)
440 - attn_mask = attn_mask * torch.finfo(query.dtype).min
441 -
442 - attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
443 - attn_weights = attn_weights + attn_mask.float()
444 - attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
445 - out = torch.einsum("hqk,khd->qhd", attn_weights, value)
446 - return out
vllm/attention/ops/triton_flash_attention.py new
+809
@@ -0,0 +1,809 @@
1 +#!/usr/bin/env python
2 +"""
3 +Fused Attention
4 +===============
5 +
6 +This is a Triton implementation of the Flash Attention v2 algorithm from Tri Dao
7 +(https://tridao.me/publications/flash2/flash2.pdf)
8 +Credits: OpenAI kernel team, AMD ML Frameworks Triton team
9 +
10 +Features supported:
11 +
12 +1) Fwd with causal masking
13 +2) Any sequence lengths without padding (currently fwd kernel only)
14 +3) Support for different sequence lengths for q and k
15 +4) Nested tensor API currently does not support dropout or bias.
16 +
17 +Not currently supported:
18 +
19 +1) Non power of two head dims
20 +
21 +"""
22 +
23 +import torch
24 +import triton
25 +import triton.language as tl
26 +
27 +torch_dtype: tl.constexpr = torch.float16
28 +
29 +
30 +@triton.jit
31 +def cdiv_fn(x, y):
32 + return (x + y - 1) // y
33 +
34 +
35 +@triton.jit
36 +def max_fn(x, y):
37 + return tl.math.max(x, y)
38 +
39 +
40 +@triton.jit
41 +def dropout_offsets(philox_seed, philox_offset, dropout_p, m, n, stride):
42 + ms = tl.arange(0, m)
43 + ns = tl.arange(0, n)
44 + return philox_offset + ms[:, None] * stride + ns[None, :]
45 +
46 +
47 +@triton.jit
48 +def dropout_rng(philox_seed, philox_offset, dropout_p, m, n, stride):
49 + rng_offsets = dropout_offsets(philox_seed, philox_offset, dropout_p, m, n,
50 + stride).to(tl.uint32)
51 + # TODO: use tl.randint for better performance
52 + return tl.rand(philox_seed, rng_offsets)
53 +
54 +
55 +@triton.jit
56 +def dropout_mask(philox_seed, philox_offset, dropout_p, m, n, stride):
57 + rng_output = dropout_rng(philox_seed, philox_offset, dropout_p, m, n,
58 + stride)
59 + rng_keep = rng_output > dropout_p
60 + return rng_keep
61 +
62 +
63 +@triton.jit
64 +def load_fn(block_ptr, first, second, pad):
65 + if first and second:
66 + tensor = tl.load(block_ptr, boundary_check=(0, 1), padding_option=pad)
67 + elif first:
68 + tensor = tl.load(block_ptr, boundary_check=(0, ), padding_option=pad)
69 + elif second:
70 + tensor = tl.load(block_ptr, boundary_check=(1, ), padding_option=pad)
71 + else:
72 + tensor = tl.load(block_ptr)
73 + return tensor
74 +
75 +
76 +@triton.jit
77 +def _attn_fwd_inner(
78 + acc,
79 + l_i,
80 + m_i,
81 + q,
82 + K_block_ptr,
83 + V_block_ptr,
84 + start_m,
85 + actual_seqlen_k,
86 + dropout_p,
87 + philox_seed,
88 + batch_philox_offset,
89 + encoded_softmax_block_ptr,
90 + block_min,
91 + block_max,
92 + offs_n_causal,
93 + masked_blocks,
94 + n_extra_tokens,
95 + bias_ptr,
96 + IS_CAUSAL: tl.constexpr,
97 + BLOCK_M: tl.constexpr,
98 + BLOCK_DMODEL: tl.constexpr,
99 + BLOCK_N: tl.constexpr,
100 + OFFS_M: tl.constexpr,
101 + OFFS_N: tl.constexpr,
102 + PRE_LOAD_V: tl.constexpr,
103 + MASK_STEPS: tl.constexpr,
104 + ENABLE_DROPOUT: tl.constexpr,
105 + RETURN_ENCODED_SOFTMAX: tl.constexpr,
106 + PADDED_HEAD: tl.constexpr,
107 +):
108 + # loop over k, v, and update accumulator
109 + for start_n in range(block_min, block_max, BLOCK_N):
110 + # For padded blocks, we will overrun the tensor size if
111 + # we load all BLOCK_N. For others, the blocks are all within range.
112 + k = load_fn(
113 + K_block_ptr,
114 + PADDED_HEAD,
115 + MASK_STEPS and (n_extra_tokens != 0),
116 + "zero",
117 + )
118 + if PRE_LOAD_V:
119 + v = load_fn(
120 + V_block_ptr,
121 + MASK_STEPS and (n_extra_tokens != 0),
122 + PADDED_HEAD,
123 + "zero",
124 + )
125 + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
126 + # We start from end of seqlen_k so only the first iteration would need
127 + # to be checked for padding if it is not a multiple of block_n
128 + # TODO: This can be optimized to only be true for the padded block.
129 + if MASK_STEPS: # noqa: SIM102
130 + # If this is the last block / iteration, we want to
131 + # mask if the sequence length is not a multiple of block size
132 + # a solution is to always do BLOCK_M // BLOCK_N + 1 steps
133 + # if not is_modulo_mn. last step might get wasted but that is okay.
134 + # check if this masking works for that case.
135 + if (start_n + BLOCK_N == block_max) and (n_extra_tokens != 0):
136 + boundary_m = tl.full([BLOCK_M],
137 + actual_seqlen_k,
138 + dtype=tl.int32)
139 + size_n = start_n + OFFS_N[None, :]
140 + mask = size_n < boundary_m[:, None]
141 + qk = tl.where(mask, qk, float("-inf"))
142 + if IS_CAUSAL:
143 + causal_boundary = start_n + offs_n_causal
144 + causal_mask = OFFS_M[:, None] >= causal_boundary[None, :]
145 + qk = tl.where(causal_mask, qk, float("-inf"))
146 + # -- compute qk ----
147 + qk += tl.dot(q, k)
148 + if bias_ptr is not None:
149 + bias = load_fn(bias_ptr, False, MASK_STEPS
150 + and (n_extra_tokens != 0), "zero")
151 + # While bias is added after multiplying qk with sm_scale, our
152 + # optimization to use 2^x instead of e^x results in an additional
153 + # scale factor of log2(e) which we must also multiply the bias with.
154 + qk += bias * 1.44269504089
155 + m_ij = tl.maximum(m_i, tl.max(qk, 1))
156 + qk = qk - m_ij[:, None]
157 + p = tl.math.exp2(qk)
158 +
159 + # CAVEAT: Must update l_ij before applying dropout
160 + l_ij = tl.sum(p, 1)
161 + if ENABLE_DROPOUT:
162 + philox_offset = (batch_philox_offset +
163 + start_m * BLOCK_M * actual_seqlen_k + start_n -
164 + BLOCK_N)
165 + keep = dropout_mask(
166 + philox_seed,
167 + philox_offset,
168 + dropout_p,
169 + BLOCK_M,
170 + BLOCK_N,
171 + actual_seqlen_k,
172 + )
173 + if RETURN_ENCODED_SOFTMAX:
174 + tl.store(
175 + encoded_softmax_block_ptr,
176 + tl.where(keep, p,
177 + -p).to(encoded_softmax_block_ptr.type.element_ty),
178 + )
179 + p = tl.where(keep, p, 0.0)
180 + elif RETURN_ENCODED_SOFTMAX:
181 + tl.store(
182 + encoded_softmax_block_ptr,
183 + p.to(encoded_softmax_block_ptr.type.element_ty),
184 + )
185 + # -- update output accumulator --
186 + alpha = tl.math.exp2(m_i - m_ij)
187 + acc = acc * alpha[:, None]
188 + if not PRE_LOAD_V:
189 + v = load_fn(
190 + V_block_ptr,
191 + MASK_STEPS and (n_extra_tokens != 0),
192 + PADDED_HEAD,
193 + "zero",
194 + )
195 + # -- update m_i and l_i
196 + l_i = l_i * alpha + l_ij
197 + # update m_i and l_i
198 + m_i = m_ij
199 + acc += tl.dot(p.to(V_block_ptr.type.element_ty), v)
200 + V_block_ptr = tl.advance(V_block_ptr, (BLOCK_N, 0))
201 + K_block_ptr = tl.advance(K_block_ptr, (0, BLOCK_N))
202 + if bias_ptr is not None:
203 + bias_ptr = tl.advance(bias_ptr, (0, BLOCK_N))
204 + if RETURN_ENCODED_SOFTMAX:
205 + encoded_softmax_block_ptr = tl.advance(encoded_softmax_block_ptr,
206 + (0, BLOCK_N))
207 + return acc, l_i, m_i
208 +
209 +
210 +@triton.autotune(
211 + configs=[
212 + triton.Config(
213 + {
214 + "BLOCK_M": 256,
215 + "BLOCK_N": 64,
216 + "waves_per_eu": 2,
217 + "PRE_LOAD_V": False,
218 + },
219 + num_stages=1,
220 + num_warps=8,
221 + ),
222 + triton.Config(
223 + {
224 + "BLOCK_M": 128,
225 + "BLOCK_N": 128,
226 + "waves_per_eu": 2,
227 + "PRE_LOAD_V": False,
228 + },
229 + num_stages=1,
230 + num_warps=4,
231 + ),
232 + triton.Config(
233 + {
234 + "BLOCK_M": 256,
235 + "BLOCK_N": 128,
236 + "waves_per_eu": 2,
237 + "PRE_LOAD_V": False,
238 + },
239 + num_stages=1,
240 + num_warps=8,
241 + ),
242 + triton.Config(
243 + {
244 + "BLOCK_M": 128,
245 + "BLOCK_N": 64,
246 + "waves_per_eu": 3,
247 + "PRE_LOAD_V": True,
248 + },
249 + num_stages=1,
250 + num_warps=4,
251 + ),
252 + triton.Config(
253 + {
254 + "BLOCK_M": 128,
255 + "BLOCK_N": 64,
256 + "waves_per_eu": 3,
257 + "PRE_LOAD_V": False,
258 + },
259 + num_stages=1,
260 + num_warps=4,
261 + ),
262 + triton.Config(
263 + {
264 + "BLOCK_M": 64,
265 + "BLOCK_N": 64,
266 + "waves_per_eu": 4,
267 + "PRE_LOAD_V": False,
268 + },
269 + num_stages=1,
270 + num_warps=8,
271 + ),
272 + triton.Config(
273 + {
274 + "BLOCK_M": 32,
275 + "BLOCK_N": 32,
276 + "waves_per_eu": 4,
277 + "PRE_LOAD_V": False,
278 + },
279 + num_stages=1,
280 + num_warps=8,
281 + ),
282 + # TODO: This config fails with head_size not pow2 with data mismatches.
283 + # triton.Config({'BLOCK_M': 32, 'BLOCK_N': 16, 'waves_per_eu': 1,
284 + # 'PRE_LOAD_V': False}, num_stages=1, num_warps=4),
285 + triton.Config(
286 + {
287 + "BLOCK_M": 16,
288 + "BLOCK_N": 16,
289 + "waves_per_eu": 1,
290 + "PRE_LOAD_V": False,
291 + },
292 + num_stages=1,
293 + num_warps=4,
294 + ),
295 + ],
296 + key=["hq", "hk", "IS_CAUSAL", "dropout_p", "BLOCK_DMODEL"],
297 +)
298 +@triton.jit
299 +def attn_fwd(
300 + Q,
301 + K,
302 + V,
303 + bias,
304 + sm_scale,
305 + L,
306 + Out,
307 + stride_qz,
308 + stride_qh,
309 + stride_qm,
310 + stride_qk,
311 + stride_kz,
312 + stride_kh,
313 + stride_kn,
314 + stride_kk,
315 + stride_vz,
316 + stride_vh,
317 + stride_vk,
318 + stride_vn,
319 + stride_oz,
320 + stride_oh,
321 + stride_om,
322 + stride_on,
323 + stride_bz,
324 + stride_bh,
325 + stride_bm,
326 + stride_bn,
327 + cu_seqlens_q,
328 + cu_seqlens_k,
329 + dropout_p,
330 + philox_seed,
331 + philox_offset_base,
332 + encoded_softmax,
333 + hq,
334 + hk,
335 + ACTUAL_BLOCK_DMODEL: tl.constexpr,
336 + MAX_SEQLENS_Q: tl.constexpr,
337 + MAX_SEQLENS_K: tl.constexpr,
338 + VARLEN: tl.constexpr,
339 + IS_CAUSAL: tl.constexpr,
340 + BLOCK_M: tl.constexpr,
341 + BLOCK_DMODEL: tl.constexpr,
342 + BLOCK_N: tl.constexpr,
343 + PRE_LOAD_V: tl.constexpr,
344 + BIAS_TYPE: tl.constexpr,
345 + ENABLE_DROPOUT: tl.constexpr,
346 + RETURN_ENCODED_SOFTMAX: tl.constexpr,
347 +):
348 + start_m = tl.program_id(0)
349 + off_h_q = tl.program_id(1)
350 + off_z = tl.program_id(2)
351 + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
352 + offs_n = tl.arange(0, BLOCK_N)
353 + if VARLEN:
354 + cu_seqlens_q_start = tl.load(cu_seqlens_q + off_z)
355 + cu_seqlens_q_end = tl.load(cu_seqlens_q + off_z + 1)
356 + seqlen_q = cu_seqlens_q_end - cu_seqlens_q_start
357 + # We have a one-size-fits-all grid in id(0). Some seqlens might be too
358 + # small for all start_m so for those we return early.
359 + if start_m * BLOCK_M > seqlen_q:
360 + return
361 + cu_seqlens_k_start = tl.load(cu_seqlens_k + off_z)
362 + cu_seqlens_k_end = tl.load(cu_seqlens_k + off_z + 1)
363 + seqlen_k = cu_seqlens_k_end - cu_seqlens_k_start
364 + else:
365 + cu_seqlens_q_start = 0
366 + cu_seqlens_k_start = 0
367 + seqlen_q = MAX_SEQLENS_Q
368 + seqlen_k = MAX_SEQLENS_K
369 +
370 + # Now we compute whether we need to exit early due to causal masking.
371 + # This is because for seqlen_q > seqlen_k, M rows of the attn scores
372 + # are completely masked, resulting in 0s written to the output, and
373 + # inf written to LSE. We don't need to do any GEMMs in this case.
374 + # This block of code determines what N is, and if this WG is operating
375 + # on those M rows.
376 + n_blocks = cdiv_fn(seqlen_k, BLOCK_N)
377 + if IS_CAUSAL:
378 + # If seqlen_q == seqlen_k, the attn scores are a square matrix.
379 + # If seqlen_q != seqlen_k, attn scores are rectangular which means
380 + # the causal mask boundary is bottom right aligned, and ends at either
381 + # the top edge (seqlen_q < seqlen_k) or left edge.
382 + # This captures the decrease in n_blocks if we have a rectangular attn
383 + # matrix
384 + n_blocks_seqlen = cdiv_fn(
385 + (start_m + 1) * BLOCK_M + seqlen_k - seqlen_q, BLOCK_N)
386 + # This is what adjusts the block_max for the current WG, only
387 + # if IS_CAUSAL. Otherwise we want to always iterate through all n_blocks
388 + n_blocks = min(n_blocks, n_blocks_seqlen)
389 + # If we have no blocks after adjusting for seqlen deltas, this WG is
390 + # part of the blocks that are all 0. We exit early.
391 + if n_blocks <= 0:
392 + o_offset = (off_z * stride_oz + cu_seqlens_q_start * stride_om +
393 + off_h_q * stride_oh)
394 + O_block_ptr = tl.make_block_ptr(
395 + base=Out + o_offset,
396 + shape=(seqlen_q, BLOCK_DMODEL),
397 + strides=(stride_om, stride_on),
398 + offsets=(start_m * BLOCK_M, 0),
399 + block_shape=(BLOCK_M, BLOCK_DMODEL),
400 + order=(1, 0),
401 + )
402 + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=Out.type.element_ty)
403 + # We still need to write 0s to the result
404 + # tl.store(O_block_ptr,
405 + # acc.to(Out.type.element_ty), boundary_check=(0,1))
406 + # l_ptrs = L + off_z * hq * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q
407 + # + offs_m
408 + # We store inf to LSE, not -inf because in the bwd pass,
409 + # we subtract this
410 + # from qk which makes it -inf, such that exp(qk - inf) = 0
411 + # for these masked blocks.
412 + # l = tl.full([BLOCK_M], value=float("inf"), dtype=tl.float32)
413 + # tl.store(l_ptrs, l)
414 + # TODO: Should dropout and return encoded softmax be handled here?
415 + return
416 +
417 + is_mqa = hq != hk
418 + off_h_k = off_h_q % hk if is_mqa else off_h_q
419 + n_extra_tokens = 0
420 + if seqlen_k < BLOCK_N:
421 + n_extra_tokens = BLOCK_N - seqlen_k
422 + elif seqlen_k % BLOCK_N:
423 + n_extra_tokens = seqlen_k % BLOCK_N
424 + padded_head = ACTUAL_BLOCK_DMODEL != BLOCK_DMODEL
425 +
426 + # Compute pointers for all the tensors used in this kernel.
427 + q_offset = (off_z * stride_qz + off_h_q * stride_qh +
428 + cu_seqlens_q_start * stride_qm)
429 + Q_block_ptr = tl.make_block_ptr(
430 + base=Q + q_offset,
431 + shape=(seqlen_q, ACTUAL_BLOCK_DMODEL),
432 + strides=(stride_qm, stride_qk),
433 + offsets=(start_m * BLOCK_M, 0),
434 + block_shape=(BLOCK_M, BLOCK_DMODEL),
435 + order=(1, 0),
436 + )
437 + k_offset = (off_z * stride_kz + off_h_k * stride_kh +
438 + cu_seqlens_k_start * stride_kn)
439 + K_block_ptr = tl.make_block_ptr(
440 + base=K + k_offset,
441 + shape=(ACTUAL_BLOCK_DMODEL, seqlen_k),
442 + strides=(stride_kk, stride_kn),
443 + offsets=(0, 0),
444 + block_shape=(BLOCK_DMODEL, BLOCK_N),
445 + order=(0, 1),
446 + )
447 + v_offset = (off_z * stride_vz + off_h_k * stride_vh +
448 + cu_seqlens_k_start * stride_vk)
449 + V_block_ptr = tl.make_block_ptr(
450 + base=V + v_offset,
451 + shape=(seqlen_k, ACTUAL_BLOCK_DMODEL),
452 + strides=(stride_vk, stride_vn),
453 + offsets=(0, 0),
454 + block_shape=(BLOCK_N, BLOCK_DMODEL),
455 + order=(1, 0),
456 + )
457 + if BIAS_TYPE != 0:
458 + bias_ptr = tl.make_block_ptr(
459 + base=bias + off_h_q * stride_bh,
460 + shape=(seqlen_q, seqlen_k),
461 + strides=(stride_bm, stride_bn),
462 + offsets=(start_m * BLOCK_M, 0),
463 + block_shape=(BLOCK_M, BLOCK_N),
464 + order=(1, 0),
465 + )
466 + else:
467 + bias_ptr = None
468 + if ENABLE_DROPOUT:
469 + batch_philox_offset = philox_offset_base \
470 + + (off_z * hq + off_h_q) \
471 + * seqlen_q * seqlen_k
472 + else:
473 + batch_philox_offset = 0
474 + # We can ask to return the dropout mask without actually doing any dropout.
475 + # In this case, we return an invalid pointer so indicate the mask is not i
476 + # valid.
477 + # TODO: Fix encoded softmax. It currently uses just h_q in the base offset.
478 + if RETURN_ENCODED_SOFTMAX:
479 + encoded_softmax_block_ptr = tl.make_block_ptr(
480 + base=encoded_softmax + off_h_q * seqlen_q * seqlen_k,
481 + shape=(seqlen_q, seqlen_k),
482 + strides=(seqlen_k, 1),
483 + offsets=(start_m * BLOCK_M, 0),
484 + block_shape=(BLOCK_M, BLOCK_N),
485 + order=(1, 0),
486 + )
487 + else:
488 + encoded_softmax_block_ptr = 0
489 + # initialize pointer to m and l
490 + m_i = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32)
491 + l_i = tl.full([BLOCK_M], 1.0, dtype=tl.float32)
492 + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
493 + # scale sm_scale by log_2(e) and use 2^x in the loop as we do not
494 + # have native e^x support in HW.
495 + qk_scale = sm_scale * 1.44269504089
496 + # Q is loaded once at the beginning and shared by all N blocks.
497 + q = load_fn(Q_block_ptr, True, padded_head, "zero")
498 + q = (q * qk_scale).to(Q_block_ptr.type.element_ty)
499 +
500 + # Here we compute how many full and masked blocks we have.
501 + padded_block_k = n_extra_tokens != 0
502 + is_modulo_mn = not padded_block_k and (seqlen_q % BLOCK_M == 0)
503 + if IS_CAUSAL:
504 + # There are always at least BLOCK_M // BLOCK_N masked blocks.
505 + # Additionally there might be one more due to dissimilar seqlens.
506 + masked_blocks = BLOCK_M // BLOCK_N + (not is_modulo_mn)
507 + else:
508 + # Padding on Q does not need to be masked in the FA loop.
509 + masked_blocks = padded_block_k
510 + # if IS_CAUSAL, not is_modulo_mn does not always result in an additional
511 + # block. In this case we might exceed n_blocks so pick the min.
512 + masked_blocks = min(masked_blocks, n_blocks)
513 + n_full_blocks = n_blocks - masked_blocks
514 + block_min = 0
515 + block_max = n_blocks * BLOCK_N
516 + # Compute for full blocks. Here we set causal to false regardless of its
517 + # value because there is no masking. Similarly we do not need padding.
518 + if n_full_blocks > 0:
519 + block_max = (n_blocks - masked_blocks) * BLOCK_N
520 + acc, l_i, m_i = _attn_fwd_inner(
521 + acc,
522 + l_i,
523 + m_i,
524 + q,
525 + K_block_ptr,
526 + V_block_ptr,
527 + start_m,
528 + seqlen_k,
529 + dropout_p,
530 + philox_seed,
531 + batch_philox_offset,
532 + encoded_softmax_block_ptr,
533 + # _, _, offs_n_causal, masked_blocks, n_extra_tokens, _
534 + block_min,
535 + block_max,
536 + 0,
537 + 0,
538 + 0,
539 + bias_ptr,
540 + # IS_CAUSAL, ....
541 + False,
542 + BLOCK_M,
543 + BLOCK_DMODEL,
544 + BLOCK_N,
545 + offs_m,
546 + offs_n,
547 + # _, MASK_STEPS, ...
548 + PRE_LOAD_V,
549 + False,
550 + ENABLE_DROPOUT,
551 + RETURN_ENCODED_SOFTMAX,
552 + padded_head,
553 + )
554 + block_min = block_max
555 + block_max = n_blocks * BLOCK_N
556 +
557 + tl.debug_barrier()
558 + # Remaining blocks, if any, are full / not masked.
559 + if masked_blocks > 0:
560 + offs_n_causal = offs_n + (seqlen_q - seqlen_k) if IS_CAUSAL else 0
561 + K_block_ptr = tl.advance(K_block_ptr, (0, n_full_blocks * BLOCK_N))
562 + V_block_ptr = tl.advance(V_block_ptr, (n_full_blocks * BLOCK_N, 0))
563 + if bias_ptr is not None:
564 + bias_ptr = tl.advance(bias_ptr, (0, n_full_blocks * BLOCK_N))
565 + if RETURN_ENCODED_SOFTMAX:
566 + encoded_softmax_block_ptr = tl.advance(encoded_softmax_block_ptr,
567 + (0, n_full_blocks))
568 + acc, l_i, m_i = _attn_fwd_inner(
569 + acc,
570 + l_i,
571 + m_i,
572 + q,
573 + K_block_ptr,
574 + V_block_ptr,
575 + start_m,
576 + seqlen_k,
577 + dropout_p,
578 + philox_seed,
579 + batch_philox_offset,
580 + encoded_softmax_block_ptr,
581 + block_min,
582 + block_max,
583 + offs_n_causal,
584 + masked_blocks,
585 + n_extra_tokens,
586 + bias_ptr,
587 + IS_CAUSAL,
588 + BLOCK_M,
589 + BLOCK_DMODEL,
590 + BLOCK_N,
591 + offs_m,
592 + offs_n,
593 + # _, MASK_STEPS, ...
594 + PRE_LOAD_V,
595 + True,
596 + ENABLE_DROPOUT,
597 + RETURN_ENCODED_SOFTMAX,
598 + padded_head,
599 + )
600 + # epilogue
601 + acc = acc / l_i[:, None]
602 + if ENABLE_DROPOUT:
603 + acc = acc / (1 - dropout_p)
604 + # If seqlen_q > seqlen_k but the delta is not a multiple of BLOCK_M,
605 + # then we have one block with a row of all NaNs which come from computing
606 + # softmax over a row of all -infs (-inf - inf = NaN). We check for that here
607 + # and store 0s where there are NaNs as these rows should've been zeroed out.
608 + end_m_idx = (start_m + 1) * BLOCK_M
609 + start_m_idx = start_m * BLOCK_M
610 + causal_start_idx = seqlen_q - seqlen_k
611 + acc = acc.to(Out.type.element_ty)
612 + if IS_CAUSAL: # noqa: SIM102
613 + if causal_start_idx > start_m_idx and causal_start_idx < end_m_idx:
614 + out_mask_boundary = tl.full((BLOCK_DMODEL, ),
615 + causal_start_idx,
616 + dtype=tl.int32)
617 + mask_m_offsets = start_m_idx + tl.arange(0, BLOCK_M)
618 + out_ptrs_mask = (mask_m_offsets[:, None] >=
619 + out_mask_boundary[None, :])
620 + z = 0.0
621 + acc = tl.where(out_ptrs_mask, acc, z.to(acc.type.element_ty))
622 + # write back LSE
623 + # l_ptrs = L + off_z * hq * MAX_SEQLENS_Q + off_h_q * MAX_SEQLENS_Q + offs_m
624 + # If seqlen_q not multiple of BLOCK_M, we need to mask out the last
625 + # few rows. This is only true for the last M block. For others,
626 + # overflow_size will be -ve
627 + # overflow_size = end_m_idx - seqlen_q
628 + # if overflow_size > 0:
629 + # boundary = tl.full((BLOCK_M,), BLOCK_M - overflow_size, dtype=tl.int32)
630 + # # This is a > check because mask being 0 blocks the store.
631 + # l_ptrs_mask = boundary > tl.arange(0, BLOCK_M)
632 + # tl.store(l_ptrs, m_i + tl.math.log2(l_i), mask=l_ptrs_mask)
633 + # else:
634 + # tl.store(l_ptrs, m_i + tl.math.log2(l_i))
635 +
636 + # write back O
637 + o_offset = (off_z * stride_oz + cu_seqlens_q_start * stride_om +
638 + off_h_q * stride_oh)
639 + O_block_ptr = tl.make_block_ptr(
640 + base=Out + o_offset,
641 + shape=(seqlen_q, ACTUAL_BLOCK_DMODEL),
642 + strides=(stride_om, stride_on),
643 + offsets=(start_m * BLOCK_M, 0),
644 + block_shape=(BLOCK_M, BLOCK_DMODEL),
645 + order=(1, 0),
646 + )
647 + # Need boundary check on this to make sure the padding from the
648 + # Q and KV tensors in both dims are not part of what we store back.
649 + # TODO: Do the boundary check optionally.
650 + tl.store(O_block_ptr, acc, boundary_check=(0, 1))
651 +
652 +
653 +def check_args(
654 + q,
655 + k,
656 + v,
657 + o,
658 + varlen=True,
659 + max_seqlens=None,
660 + cu_seqlens_q=None,
661 + cu_seqlens_k=None,
662 +):
663 + assert q.dim() == k.dim() and q.dim() == v.dim()
664 + if varlen:
665 + assert q.dim() == 3
666 + total_q, nheads_q, head_size = q.shape
667 + total_k, nheads_k, _ = k.shape
668 + assert cu_seqlens_q is not None
669 + assert cu_seqlens_k is not None
670 + assert len(cu_seqlens_q) == len(cu_seqlens_k)
671 + else:
672 + assert q.dim() == 4
673 + batch, nheads_q, seqlen_q, head_size = q.shape
674 + _, nheads_k, seqlen_k, _ = k.shape
675 + assert max_seqlens > 0
676 + assert k.shape == v.shape
677 + assert q.shape[-1] == k.shape[-1] and q.shape[-1] == v.shape[-1]
678 + # TODO: Change assert if we support qkl f8 and v f16
679 + assert q.dtype == k.dtype and q.dtype == v.dtype
680 + # TODO: Fix assert to check head size <=256 once supported
681 + assert head_size <= 128
682 + assert o.shape == q.shape
683 + assert (nheads_q % nheads_k) == 0
684 +
685 +
686 +class _attention(torch.autograd.Function):
687 +
688 + @staticmethod
689 + def forward(
690 + ctx,
691 + q,
692 + k,
693 + v,
694 + o,
695 + cu_seqlens_q,
696 + cu_seqlens_k,
697 + max_seqlens_q,
698 + max_seqlens_k,
699 + causal=False,
700 + sm_scale=1.0,
701 + bias=None,
702 + ):
703 + if o is None:
704 + o = torch.empty_like(q, dtype=v.dtype)
705 +
706 + check_args(
707 + q,
708 + k,
709 + v,
710 + o,
711 + varlen=True,
712 + cu_seqlens_q=cu_seqlens_q,
713 + cu_seqlens_k=cu_seqlens_k,
714 + )
715 + if True: # varlen
716 + total_q, nheads_q, head_size = q.shape
717 + total_k, nheads_k, _ = k.shape
718 + batch = len(cu_seqlens_q) - 1
719 + q_strides = (0, q.stride(1), q.stride(0), q.stride(2))
720 + k_strides = (0, k.stride(1), k.stride(0), k.stride(2))
721 + v_strides = (0, v.stride(1), v.stride(0), v.stride(2))
722 + o_strides = (0, o.stride(1), o.stride(0), o.stride(2))
723 + else:
724 + batch, seqlen_q, nheads_q, head_size = q.shape
725 + _, seqlen_k, nheads_k, _ = k.shape
726 + q_strides = (q.stride(0), q.stride(2), q.stride(1), q.stride(3))
727 + k_strides = (k.stride(0), k.stride(2), k.stride(1), k.stride(3))
728 + v_strides = (v.stride(0), v.stride(2), v.stride(1), v.stride(3))
729 + o_strides = (o.stride(0), o.stride(2), o.stride(1), o.stride(3))
730 +
731 + # Get closest power of 2 over or equal to 32.
732 + unpadded_head_dims = {32, 64, 128}
733 + if head_size not in unpadded_head_dims:
734 + padded_d_model = None
735 + for i in unpadded_head_dims:
736 + if i > head_size:
737 + padded_d_model = i
738 + break
739 + assert padded_d_model is not None
740 + else:
741 + padded_d_model = head_size
742 +
743 + grid = lambda META: (
744 + triton.cdiv(max_seqlens_q, META["BLOCK_M"]),
745 + nheads_q,
746 + batch,
747 + )
748 +
749 + encoded_softmax = None
750 +
751 + # Seed the RNG so we get reproducible results for testing.
752 + philox_seed = 0x1BF52
753 + philox_offset = 0x1D4B42
754 +
755 + if bias is not None:
756 + bias_strides = (
757 + bias.stride(0),
758 + bias.stride(1),
759 + bias.stride(2),
760 + bias.stride(3),
761 + )
762 + else:
763 + bias_strides = (0, 0, 0, 0)
764 +
765 + attn_fwd[grid](
766 + q,
767 + k,
768 + v,
769 + bias,
770 + sm_scale,
771 + None,
772 + o,
773 + *q_strides,
774 + *k_strides,
775 + *v_strides,
776 + *o_strides,
777 + *bias_strides,
778 + cu_seqlens_q,
779 + cu_seqlens_k,
780 + dropout_p=0.0,
781 + philox_seed=philox_seed,
782 + philox_offset_base=philox_offset,
783 + encoded_softmax=encoded_softmax,
784 + hq=nheads_q,
785 + hk=nheads_k,
786 + ACTUAL_BLOCK_DMODEL=head_size,
787 + MAX_SEQLENS_Q=max_seqlens_q,
788 + MAX_SEQLENS_K=max_seqlens_k,
789 + IS_CAUSAL=causal,
790 + VARLEN=True,
791 + BLOCK_DMODEL=padded_d_model,
792 + BIAS_TYPE=0 if bias is None else 1,
793 + ENABLE_DROPOUT=False,
794 + RETURN_ENCODED_SOFTMAX=False,
795 + )
796 +
797 + ctx.grid = grid
798 + ctx.sm_scale = sm_scale
799 + ctx.BLOCK_DMODEL = head_size
800 + ctx.causal = causal
801 + ctx.dropout_p = 0.0
802 + ctx.philox_seed = philox_seed
803 + ctx.philox_offset = philox_offset
804 + ctx.encoded_softmax = encoded_softmax
805 + ctx.return_encoded_softmax = False
806 + return o, encoded_softmax
807 +
808 +
809 +triton_attention = _attention.apply
vllm/attention/selector.py
+40 -17
@@ -1,3 +1,4 @@
1 +import enum
2 from functools import lru_cache
3 from typing import Type
4
@@ -10,46 +11,68 @@ from vllm.utils import is_cpu, is_hip
11 logger = init_logger(__name__)
12
13
14 +class _Backend(enum.Enum):
15 + FLASH_ATTN = enum.auto()
16 + XFORMERS = enum.auto()
17 + ROCM_FLASH = enum.auto()
18 + TORCH_SDPA = enum.auto()
19 +
20 +
21 @lru_cache(maxsize=None)
22 def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
15 - if _can_use_flash_attn(dtype):
23 + backend = _which_attn_to_use(dtype)
24 + if backend == _Backend.FLASH_ATTN:
25 logger.info("Using FlashAttention backend.")
26 from vllm.attention.backends.flash_attn import ( # noqa: F401
27 FlashAttentionBackend)
28 return FlashAttentionBackend
20 - elif is_cpu():
21 - logger.info("Using Torch SDPA backend.")
22 - from vllm.attention.backends.torch_sdpa import TorchSDPABackend
23 - return TorchSDPABackend
24 - else:
29 + elif backend == _Backend.XFORMERS:
30 logger.info("Using XFormers backend.")
31 from vllm.attention.backends.xformers import ( # noqa: F401
32 XFormersBackend)
33 return XFormersBackend
34 + elif backend == _Backend.ROCM_FLASH:
35 + logger.info("Using ROCmFlashAttention backend.")
36 + from vllm.attention.backends.rocm_flash_attn import ( # noqa: F401
37 + ROCmFlashAttentionBackend)
38 + return ROCmFlashAttentionBackend
39 + elif backend == _Backend.TORCH_SDPA:
40 + logger.info("Using Torch SDPA backend.")
41 + from vllm.attention.backends.torch_sdpa import TorchSDPABackend
42 + return TorchSDPABackend
43 + else:
44 + raise ValueError("Invalid attention backend.")
45
46
31 -def _can_use_flash_attn(dtype: torch.dtype) -> bool:
47 +def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
48 + """Returns which flash attention backend to use."""
49 + if is_cpu():
50 + return _Backend.TORCH_SDPA
51 +
52 if is_hip():
53 # AMD GPUs.
34 - logger.info("Cannot use FlashAttention backend for AMD GPUs.")
35 - return False
36 - if is_cpu():
37 - return False
54 + if torch.cuda.get_device_capability()[0] != 9:
55 + # not Instinct series GPUs.
56 + logger.info("flash_atten is not supported on NAVI GPUs.")
57 + return _Backend.ROCM_FLASH
58 +
59 + # NVIDIA GPUs.
60 if torch.cuda.get_device_capability()[0] < 8:
61 # Volta and Turing NVIDIA GPUs.
62 logger.info("Cannot use FlashAttention backend for Volta and Turing "
63 "GPUs.")
42 - return False
64 + return _Backend.XFORMERS
65 +
66 if dtype not in (torch.float16, torch.bfloat16):
67 logger.info("Cannot use FlashAttention backend for dtype other than "
68 "torch.float16 or torch.bfloat16.")
46 - return False
69 + return _Backend.XFORMERS
70
71 try:
72 import flash_attn # noqa: F401
73 except ImportError:
74 logger.info(
52 - "Cannot use FlashAttention because the package is not found. "
53 - "Please install it for better performance.")
54 - return False
55 - return True
75 + "Cannot use FlashAttention backend because the flash_attn package "
76 + "is not found. Please install it for better performance.")
77 + return _Backend.XFORMERS
78 + return _Backend.FLASH_ATTN