A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Misc] Add FA2 support to ViT MHA layer (#12355)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jan 25, 2025 at 15:07 UTC
f1fc0510dfbb11c98f41d02a44e092785c626314
2 files changed
+146
-5
tests/kernels/test_mha_attn.py
new
+126
@@ -0,0 +1,126 @@
1
+"""
2
+Test:
3
+
4
+* Tests for MultiHeadAttention layer
5
+"""
6
+from unittest.mock import patch
7
+
8
+import pytest
9
+import torch
10
+
11
+from vllm.attention.layer import MultiHeadAttention
12
+from vllm.attention.selector import _Backend, _cached_get_attn_backend
13
+from vllm.platforms import current_platform
14
+from vllm.platforms.cpu import CpuPlatform
15
+from vllm.platforms.cuda import CudaPlatform
16
+from vllm.platforms.rocm import RocmPlatform
17
+
18
+
19
+@pytest.fixture(autouse=True)
20
+def clear_cache():
21
+ """Clear lru cache to ensure each test case runs without caching.
22
+ """
23
+ _cached_get_attn_backend.cache_clear()
24
+
25
+
26
+@pytest.mark.parametrize("device", ["cpu", "hip", "cuda"])
27
+def test_mha_attn_platform(device: str):
28
+ """
29
+ Test that the attention selector between different platform and device.
30
+ """
31
+ torch.set_default_dtype(torch.float16)
32
+
33
+ if device == "cpu":
34
+ with patch("vllm.attention.selector.current_platform", CpuPlatform()):
35
+ attn = MultiHeadAttention(16, 64, scale=1)
36
+ assert attn.attn_backend == _Backend.TORCH_SDPA
37
+ elif device == "hip":
38
+ with patch("vllm.attention.selector.current_platform", RocmPlatform()):
39
+ attn = MultiHeadAttention(16, 64, scale=1)
40
+ assert attn.attn_backend == _Backend.TORCH_SDPA
41
+ else:
42
+ with patch("vllm.attention.selector.current_platform", CudaPlatform()):
43
+ attn = MultiHeadAttention(16, 64, scale=1)
44
+ assert attn.attn_backend == _Backend.FLASH_ATTN
45
+
46
+ with patch("vllm.attention.selector.current_platform", CudaPlatform()):
47
+ attn = MultiHeadAttention(16, 72, scale=1)
48
+ assert attn.attn_backend == _Backend.XFORMERS
49
+
50
+
51
+def ref_attention(
52
+ query: torch.Tensor,
53
+ key: torch.Tensor,
54
+ value: torch.Tensor,
55
+ scale: float,
56
+) -> torch.Tensor:
57
+ """
58
+ Native implementation of scaled dot product attention without mask:
59
+ - query, key, value: [batch_size, seq_len, num_heads, head_size]
60
+ - attn_mask: [batch_size, seq_len, seq_len]
61
+ """
62
+ query, key, value = (x.transpose(1, 2) for x in (query, key, value))
63
+ attn_weights = scale * torch.matmul(query, key.transpose(2, 3))
64
+ attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
65
+ out = torch.matmul(attn_weights, value).transpose(1, 2)
66
+ return out
67
+
68
+
69
+BATCH_SIZES = [1, 16]
70
+SEQ_LENS = [1]
71
+NUM_HEADS = [1, 16]
72
+NUM_KV_HEADS = [1]
73
+HEAD_SIZES = [64, 80]
74
+# flshattF and tritonflashattF supported: {torch.float16, torch.bfloat16}
75
+DTYPES = [
76
+ torch.half, torch.bfloat16, torch.float
77
+] if not current_platform.is_rocm() else [torch.half, torch.bfloat16]
78
+CUDA_DEVICES = ["cuda"]
79
+
80
+
81
+@pytest.mark.parametrize("batch_size", BATCH_SIZES)
82
+@pytest.mark.parametrize("seq_len", SEQ_LENS)
83
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
84
+@pytest.mark.parametrize("num_kv_heads", NUM_KV_HEADS)
85
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
86
+@pytest.mark.parametrize("dtype", DTYPES)
87
+@pytest.mark.parametrize("device", CUDA_DEVICES)
88
+def test_mha_attn_forward(
89
+ batch_size: int,
90
+ seq_len: int,
91
+ num_heads: int,
92
+ num_kv_heads: int,
93
+ head_size: int,
94
+ dtype: torch.dtype,
95
+ device: str,
96
+):
97
+ current_platform.seed_everything(0)
98
+ torch.set_default_device(device)
99
+ torch.set_default_dtype(dtype)
100
+
101
+ q = torch.randn(batch_size, seq_len, num_heads * head_size)
102
+ k = torch.randn(batch_size, seq_len, num_kv_heads * head_size)
103
+ v = torch.randn(batch_size, seq_len, num_kv_heads * head_size)
104
+ scale = 1.0 / head_size**0.5
105
+ attn = MultiHeadAttention(num_heads,
106
+ head_size,
107
+ scale=scale,
108
+ num_kv_heads=num_kv_heads)
109
+ output = attn(q, k, v)
110
+
111
+ assert num_heads % num_kv_heads == 0
112
+ num_queries_per_kv = num_heads // num_kv_heads
113
+ q = q.reshape(batch_size, seq_len, num_heads, head_size)
114
+ k = k.reshape(batch_size, seq_len, num_kv_heads, head_size)
115
+ v = v.reshape(batch_size, seq_len, num_kv_heads, head_size)
116
+ if num_queries_per_kv > 1:
117
+ k = torch.repeat_interleave(k, num_queries_per_kv, dim=2)
118
+ v = torch.repeat_interleave(v, num_queries_per_kv, dim=2)
119
+
120
+ ref_output = ref_attention(
121
+ q,
122
+ k,
123
+ v,
124
+ scale=scale,
125
+ ).reshape(batch_size, seq_len, num_heads * head_size)
126
+ torch.testing.assert_close(output, ref_output)
vllm/attention/layer.py
+20
-5
@@ -210,6 +210,9 @@ class MultiHeadAttention(nn.Module):
210
self.scale = scale
211
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
212
213
+ assert self.num_heads % self.num_kv_heads == 0
214
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
215
+
216
dtype = torch.get_default_dtype()
217
attn_backend = get_attn_backend(head_size,
218
dtype,
@@ -217,11 +220,12 @@ class MultiHeadAttention(nn.Module):
220
block_size=16,
221
is_attention_free=False)
222
backend = backend_name_to_enum(attn_backend.get_name())
220
- if backend in {_Backend.FLASH_ATTN, _Backend.FLASH_ATTN_VLLM_V1}:
221
- backend = _Backend.XFORMERS
223
224
self.attn_backend = backend if backend in {
224
- _Backend.TORCH_SDPA, _Backend.XFORMERS
225
+ _Backend.TORCH_SDPA,
226
+ _Backend.XFORMERS,
227
+ _Backend.FLASH_ATTN,
228
+ _Backend.FLASH_ATTN_VLLM_V1,
229
} else _Backend.TORCH_SDPA
230
231
def forward(
@@ -231,7 +235,6 @@ class MultiHeadAttention(nn.Module):
235
value: torch.Tensor,
236
) -> torch.Tensor:
237
"""Input shape: batch_size x seq_len x hidden_size"""
234
- # TODO(Isotr0py): Use existing backend implementations and support FA2
238
bsz, q_len, _ = query.size()
239
kv_len = key.size(1)
240
@@ -239,7 +242,19 @@ class MultiHeadAttention(nn.Module):
242
key = key.view(bsz, kv_len, self.num_kv_heads, self.head_size)
243
value = value.view(bsz, kv_len, self.num_kv_heads, self.head_size)
244
242
- if self.attn_backend == _Backend.XFORMERS:
245
+ if (num_repeat := self.num_queries_per_kv) > 1:
246
+ # Handle MQA and GQA
247
+ key = torch.repeat_interleave(key, num_repeat, dim=2)
248
+ value = torch.repeat_interleave(value, num_repeat, dim=2)
249
+
250
+ if self.attn_backend in {
251
+ _Backend.FLASH_ATTN,
252
+ _Backend.FLASH_ATTN_VLLM_V1,
253
+ }:
254
+ from vllm.vllm_flash_attn import flash_attn_func
255
+
256
+ out = flash_attn_func(query, key, value, softmax_scale=self.scale)
257
+ elif self.attn_backend == _Backend.XFORMERS:
258
from xformers import ops as xops
259
260
out = xops.memory_efficient_attention_forward(query,