A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Clean up kernel unit tests (#938)
Woosuk Kwon committed
Sep 6, 2023 at 08:57 UTC
fbd80ad4092c4bc48ce672f0435c1d1362aee052
6 files changed
+364
-399
tests/kernels/conftest.py
new
+43
@@ -0,0 +1,43 @@
1
+from typing import List, Tuple
2
+
3
+import pytest
4
+import torch
5
+
6
+
7
+def create_kv_caches(
8
+ num_blocks: int,
9
+ block_size: int,
10
+ num_layers: int,
11
+ num_heads: int,
12
+ head_size: int,
13
+ dtype: torch.dtype,
14
+ seed: int,
15
+) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
16
+ torch.random.manual_seed(seed)
17
+ torch.cuda.manual_seed(seed)
18
+
19
+ scale = head_size**-0.5
20
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
21
+ key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
22
+ key_caches = []
23
+ for _ in range(num_layers):
24
+ key_cache = torch.empty(size=key_cache_shape,
25
+ dtype=dtype,
26
+ device='cuda')
27
+ key_cache.uniform_(-scale, scale)
28
+ key_caches.append(key_cache)
29
+
30
+ value_cache_shape = (num_blocks, num_heads, head_size, block_size)
31
+ value_caches = []
32
+ for _ in range(num_layers):
33
+ value_cache = torch.empty(size=value_cache_shape,
34
+ dtype=dtype,
35
+ device='cuda')
36
+ value_cache.uniform_(-scale, scale)
37
+ value_caches.append(value_cache)
38
+ return key_caches, value_caches
39
+
40
+
41
+@pytest.fixture()
42
+def kv_cache_factory():
43
+ return create_kv_caches
tests/kernels/test_activation.py
+31
-28
@@ -1,20 +1,34 @@
1
+import pytest
2
import torch
3
import torch.nn.functional as F
4
from transformers.activations import get_activation
5
+
6
from vllm import activation_ops
7
8
+DTYPES = [torch.half, torch.bfloat16, torch.float]
9
+NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
10
+D = [512, 4096, 5120, 13824] # Arbitrary values for testing
11
+SEEDS = [0]
12
+
13
14
def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
15
x1, x2 = x.chunk(chunks=2, dim=1)
16
return F.silu(x1) * x2
17
18
19
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
20
+@pytest.mark.parametrize("d", D)
21
+@pytest.mark.parametrize("dtype", DTYPES)
22
+@pytest.mark.parametrize("seed", SEEDS)
23
@torch.inference_mode()
13
-def run_silu_and_mul(
24
+def test_silu_and_mul(
25
num_tokens: int,
26
d: int,
27
dtype: torch.dtype,
28
+ seed: int,
29
) -> None:
30
+ torch.random.manual_seed(seed)
31
+ torch.cuda.manual_seed(seed)
32
x = torch.randn(num_tokens, 2 * d, dtype=dtype, device='cuda')
33
out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
34
activation_ops.silu_and_mul(out, x)
@@ -22,20 +36,19 @@ def run_silu_and_mul(
36
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
37
38
25
-def test_silu_and_mul() -> None:
26
- for dtype in [torch.half, torch.bfloat16, torch.float]:
27
- for num_tokens in [7, 83, 2048]:
28
- for d in [512, 4096, 5120, 13824]:
29
- print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
30
- run_silu_and_mul(num_tokens, d, dtype)
31
-
32
-
39
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
40
+@pytest.mark.parametrize("d", D)
41
+@pytest.mark.parametrize("dtype", DTYPES)
42
+@pytest.mark.parametrize("seed", SEEDS)
43
@torch.inference_mode()
34
-def run_gelu_new(
44
+def test_gelu_new(
45
num_tokens: int,
46
d: int,
47
dtype: torch.dtype,
48
+ seed: int,
49
) -> None:
50
+ torch.random.manual_seed(seed)
51
+ torch.cuda.manual_seed(seed)
52
x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
53
out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
54
activation_ops.gelu_new(out, x)
@@ -43,30 +56,20 @@ def run_gelu_new(
56
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
57
58
46
-def test_gelu_new() -> None:
47
- for dtype in [torch.half, torch.bfloat16, torch.float]:
48
- for num_tokens in [7, 83, 2048]:
49
- for d in [512, 4096, 5120, 13824]:
50
- print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
51
- run_gelu_new(num_tokens, d, dtype)
52
-
53
-
54
-@torch.inference_mode()
55
-def run_gelu_fast(
59
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
60
+@pytest.mark.parametrize("d", D)
61
+@pytest.mark.parametrize("dtype", DTYPES)
62
+@pytest.mark.parametrize("seed", SEEDS)
63
+def test_gelu_fast(
64
num_tokens: int,
65
d: int,
66
dtype: torch.dtype,
67
+ seed: int,
68
) -> None:
69
+ torch.random.manual_seed(seed)
70
+ torch.cuda.manual_seed(seed)
71
x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
72
out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
73
activation_ops.gelu_fast(out, x)
74
ref_out = get_activation("gelu_fast")(x)
75
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
65
-
66
-
67
-def test_gelu_fast() -> None:
68
- for dtype in [torch.half, torch.bfloat16, torch.float]:
69
- for num_tokens in [7, 83, 2048]:
70
- for d in [512, 4096, 5120, 13824]:
71
- print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
72
- run_gelu_fast(num_tokens, d, dtype)
tests/kernels/test_attention.py
+159
-198
@@ -1,14 +1,24 @@
1
import random
2
-from typing import List, Optional
2
+from typing import List, Optional, Tuple
3
4
+import pytest
5
import torch
6
from xformers import ops as xops
7
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9
from vllm import attention_ops
10
10
-MAX_SEQ_LEN = 4096
11
-TEST_SEED = 0
11
+MAX_SEQ_LEN = 8192
12
+NUM_BLOCKS = 128 # Arbitrary values for testing
13
+
14
+DTYPES = [torch.half, torch.bfloat16, torch.float]
15
+NUM_GEN_SEQS = [7] # Arbitrary values for testing
16
+NUM_PREFILL_SEQS = [1, 3, 7] # Arbitrary values for testing
17
+NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
18
+HEAD_SIZES = [64, 80, 96, 112, 128, 256]
19
+BLOCK_SIZES = [8, 16, 32]
20
+USE_ALIBI = [False] # TODO(woosuk): Add USE_ALIBI=True
21
+SEEDS = [0]
22
23
24
def ref_masked_attention(
@@ -18,29 +28,34 @@ def ref_masked_attention(
28
scale: float,
29
attn_mask: Optional[torch.Tensor] = None,
30
) -> torch.Tensor:
21
- query = query * scale
22
- attn = torch.einsum('qhd,khd->hqk', query, key)
31
+ attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
32
if attn_mask is not None:
24
- attn = attn + attn_mask
25
- attn = torch.softmax(attn, dim=-1)
26
- out = torch.einsum('hqk,khd->qhd', attn, value)
33
+ attn_weights = attn_weights + attn_mask.float()
34
+ attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
35
+ out = torch.einsum("hqk,khd->qhd", attn_weights, value)
36
return out
37
38
39
def ref_single_query_cached_kv_attention(
40
output: torch.Tensor,
41
query: torch.Tensor,
42
+ num_queries_per_kv: int,
43
key_cache: torch.Tensor,
44
value_cache: torch.Tensor,
45
block_tables: torch.Tensor,
46
context_lens: torch.Tensor,
47
+ scale: float,
48
+ alibi_slopes: Optional[torch.Tensor],
49
) -> None:
38
- num_heads = value_cache.shape[1]
50
+ num_query_heads = query.shape[1]
51
+ num_kv_heads = value_cache.shape[1]
52
head_size = value_cache.shape[2]
53
block_size = value_cache.shape[3]
54
+ num_seqs = query.shape[0]
55
42
- num_input_tokens = query.shape[0]
43
- for i in range(num_input_tokens):
56
+ block_tables = block_tables.cpu().tolist()
57
+ context_lens = context_lens.cpu().tolist()
58
+ for i in range(num_seqs):
59
q = query[i].unsqueeze(0)
60
block_table = block_tables[i]
61
context_len = int(context_lens[i])
@@ -52,170 +67,96 @@ def ref_single_query_cached_kv_attention(
67
block_offset = j % block_size
68
69
k = key_cache[block_number, :, :, block_offset, :]
55
- k = k.reshape(num_heads, head_size)
70
+ k = k.reshape(num_kv_heads, head_size)
71
keys.append(k)
72
73
v = value_cache[block_number, :, :, block_offset]
74
values.append(v)
75
keys = torch.stack(keys, dim=0)
76
values = torch.stack(values, dim=0)
62
-
63
- scale = 1.0 / (head_size**0.5)
64
- out = ref_masked_attention(q, keys, values, scale)
65
- out = out.view(num_heads, head_size)
77
+ if num_queries_per_kv > 1:
78
+ # Handle MQA and GQA
79
+ keys = torch.repeat_interleave(keys, num_queries_per_kv, dim=1)
80
+ values = torch.repeat_interleave(values, num_queries_per_kv, dim=1)
81
+
82
+ alibi_bias = None
83
+ if alibi_slopes is not None:
84
+ # Create the ALiBi bias used in the paged attention kernel.
85
+ position_ids = torch.arange(context_len, device="cuda").int()
86
+ alibi_bias = (context_len - position_ids).float()
87
+ alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
88
+ 1, 1, -1)
89
+
90
+ out = ref_masked_attention(q, keys, values, scale, alibi_bias)
91
+ out = out.view(num_query_heads, head_size)
92
output[i].copy_(out, non_blocking=True)
93
94
69
-def ref_multi_query_kv_attention(
70
- cu_seq_lens: List[int],
71
- query: torch.Tensor,
72
- key: torch.Tensor,
73
- value: torch.Tensor,
74
- dtype: torch.dtype,
75
-) -> torch.Tensor:
76
- head_size = query.shape[-1]
77
- scale = 1.0 / (head_size**0.5)
78
-
79
- num_seqs = len(cu_seq_lens) - 1
80
- ref_outputs = []
81
- for i in range(num_seqs):
82
- start_idx = cu_seq_lens[i]
83
- end_idx = cu_seq_lens[i + 1]
84
- seq_len = end_idx - start_idx
85
-
86
- # Create attention mask.
87
- attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
88
- diagonal=1)
89
- attn_mask = attn_mask * torch.finfo(dtype).min
90
- attn_mask = attn_mask.to(dtype=dtype, device='cuda')
91
-
92
- ref_output = ref_masked_attention(
93
- query[start_idx:end_idx],
94
- key[start_idx:end_idx],
95
- value[start_idx:end_idx],
96
- scale,
97
- attn_mask=attn_mask,
98
- )
99
- ref_outputs.append(ref_output)
100
- ref_output = torch.cat(ref_outputs, dim=0)
101
- return ref_output
102
-
103
-
104
-def ref_multi_query_cached_kv_attention(
105
- cu_query_lens: List[int],
106
- query: torch.Tensor,
107
- key_cache: torch.Tensor,
108
- value_cache: torch.Tensor,
109
- block_tables: torch.Tensor,
110
- context_lens: torch.Tensor,
111
- dtype: torch.dtype,
112
-) -> torch.Tensor:
113
- num_heads = value_cache.shape[1]
114
- head_size = value_cache.shape[2]
115
- block_size = value_cache.shape[3]
116
- scale = 1.0 / (head_size**0.5)
117
-
118
- num_queries = len(cu_query_lens) - 1
119
- ref_outputs = []
120
- for i in range(num_queries):
121
- start_idx = cu_query_lens[i]
122
- end_idx = cu_query_lens[i + 1]
123
- query_len = end_idx - start_idx
124
- context_len = int(context_lens[i])
125
- block_table = block_tables[i]
126
-
127
- # Create attention mask
128
- attn_mask = torch.triu(torch.ones(query_len, context_len),
129
- diagonal=context_len - query_len + 1) * -1e5
130
- attn_mask = attn_mask.to(dtype=dtype, device='cuda')
131
-
132
- keys = []
133
- values = []
134
- for j in range(context_len):
135
- block_number = int(block_table[j // block_size])
136
- block_offset = j % block_size
137
-
138
- k = key_cache[block_number, :, :, block_offset, :]
139
- k = k.reshape(num_heads, head_size)
140
- keys.append(k)
141
-
142
- v = value_cache[block_number, :, :, block_offset]
143
- values.append(v)
144
- keys = torch.stack(keys, dim=0)
145
- values = torch.stack(values, dim=0)
146
-
147
- ref_output = ref_masked_attention(
148
- query[start_idx:end_idx],
149
- keys,
150
- values,
151
- scale,
152
- attn_mask=attn_mask,
153
- )
154
- ref_outputs.append(ref_output)
155
- ref_output = torch.cat(ref_outputs, dim=0)
156
- return ref_output
157
-
158
-
95
+@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
96
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
97
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
98
+@pytest.mark.parametrize("use_alibi", USE_ALIBI)
99
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
100
+@pytest.mark.parametrize("dtype", DTYPES)
101
+@pytest.mark.parametrize("seed", SEEDS)
102
@torch.inference_mode()
160
-def run_single_query_cached_kv_attention(
161
- num_tokens: int,
162
- num_heads: int,
103
+def test_single_query_cached_kv_attention(
104
+ kv_cache_factory,
105
+ num_seqs: int,
106
+ num_heads: Tuple[int, int],
107
head_size: int,
108
+ use_alibi: bool,
109
block_size: int,
165
- num_blocks: int,
110
dtype: torch.dtype,
167
- num_kv_heads: int = None,
111
+ seed: int,
112
) -> None:
169
- qkv = torch.empty(num_tokens,
170
- 3,
171
- num_heads,
172
- head_size,
173
- dtype=dtype,
174
- device='cuda')
175
- qkv.uniform_(-1e-3, 1e-3)
176
- query, _, _ = qkv.unbind(dim=1)
177
-
178
- x = 16 // torch.tensor([], dtype=dtype).element_size()
179
- key_block_shape = (num_heads, head_size // x, block_size, x)
180
- key_cache = torch.empty(size=(num_blocks, *key_block_shape),
181
- dtype=dtype,
182
- device='cuda')
183
- key_cache.uniform_(-1e-3, 1e-3)
184
- value_block_shape = (num_heads, head_size, block_size)
185
- value_cache = torch.empty(size=(num_blocks, *value_block_shape),
186
- dtype=dtype,
187
- device='cuda')
188
- value_cache.uniform_(-1e-3, 1e-3)
189
-
190
- context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
113
+ random.seed(seed)
114
+ torch.random.manual_seed(seed)
115
+ torch.cuda.manual_seed(seed)
116
+
117
+ scale = float(1.0 / (head_size**0.5))
118
+ num_query_heads, num_kv_heads = num_heads
119
+ query = torch.empty(num_seqs,
120
+ num_query_heads,
121
+ head_size,
122
+ dtype=dtype,
123
+ device="cuda")
124
+ query.uniform_(-scale, scale)
125
+
126
+ assert num_query_heads % num_kv_heads == 0
127
+ num_queries_per_kv = num_query_heads // num_kv_heads
128
+ head_mapping = torch.repeat_interleave(
129
+ torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
130
+ num_queries_per_kv)
131
+ alibi_slopes = None
132
+ if use_alibi:
133
+ alibi_slopes = torch.randn(num_query_heads,
134
+ dtype=torch.float,
135
+ device="cuda")
136
+
137
+ context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
138
max_context_len = max(context_lens)
192
- context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
139
+ context_lens = torch.tensor(context_lens, dtype=torch.int, device="cuda")
140
141
+ # Create the block tables.
142
max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
143
block_tables = []
196
- for _ in range(num_tokens):
144
+ for _ in range(num_seqs):
145
block_table = [
198
- random.randint(0, num_blocks - 1)
146
+ random.randint(0, NUM_BLOCKS - 1)
147
for _ in range(max_num_blocks_per_seq)
148
]
149
block_tables.append(block_table)
202
- block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
203
- head_mapping = torch.arange(num_heads, dtype=torch.int32, device="cuda")
150
+ block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
151
205
- scale = float(1.0 / (head_size**0.5))
206
-
207
- num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
208
- assert num_heads % num_kv_heads == 0
209
- num_queries_per_kv = num_heads // num_kv_heads
210
- head_mapping = torch.repeat_interleave(
211
- torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
212
- num_queries_per_kv)
152
+ # Create the KV caches.
153
+ key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
154
+ num_kv_heads, head_size, dtype,
155
+ seed)
156
+ key_cache, value_cache = key_caches[0], value_caches[0]
157
214
- output = torch.empty(num_tokens,
215
- num_heads,
216
- head_size,
217
- dtype=dtype,
218
- device='cuda')
158
+ # Call the paged attention kernel.
159
+ output = torch.empty_like(query)
160
attention_ops.single_query_cached_kv_attention(
161
output,
162
query,
@@ -227,45 +168,98 @@ def run_single_query_cached_kv_attention(
168
context_lens,
169
block_size,
170
max_context_len,
230
- None, # ALiBi slopes.
171
+ alibi_slopes,
172
)
173
174
+ # Run the reference implementation.
175
ref_output = torch.empty_like(query)
176
ref_single_query_cached_kv_attention(
177
ref_output,
178
query,
179
+ num_queries_per_kv,
180
key_cache,
181
value_cache,
182
block_tables,
183
context_lens,
184
+ scale,
185
+ alibi_slopes,
186
)
242
- # NOTE(woosuk): Due to the difference in the data types the two
243
- # implementations use for attention softmax logits and accumulation,
244
- # there is a small difference in the final outputs.
245
- # We should use a relaxed tolerance for the test.
187
+
188
+ # NOTE(woosuk): Due to the kernel-level differences in the two
189
+ # implementations, there is a small numerical difference in the two
190
+ # outputs. Thus, we use a relaxed tolerance for the test.
191
assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
192
193
194
+def ref_multi_query_kv_attention(
195
+ cu_seq_lens: List[int],
196
+ query: torch.Tensor,
197
+ key: torch.Tensor,
198
+ value: torch.Tensor,
199
+ scale: float,
200
+ dtype: torch.dtype,
201
+) -> torch.Tensor:
202
+ num_seqs = len(cu_seq_lens) - 1
203
+ ref_outputs = []
204
+ for i in range(num_seqs):
205
+ start_idx = cu_seq_lens[i]
206
+ end_idx = cu_seq_lens[i + 1]
207
+ seq_len = end_idx - start_idx
208
+
209
+ # Create attention mask.
210
+ attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
211
+ diagonal=1)
212
+ attn_mask = attn_mask * torch.finfo(dtype).min
213
+ attn_mask = attn_mask.to(dtype=dtype, device="cuda")
214
+
215
+ ref_output = ref_masked_attention(
216
+ query[start_idx:end_idx],
217
+ key[start_idx:end_idx],
218
+ value[start_idx:end_idx],
219
+ scale,
220
+ attn_mask=attn_mask,
221
+ )
222
+ ref_outputs.append(ref_output)
223
+ ref_output = torch.cat(ref_outputs, dim=0)
224
+ return ref_output
225
+
226
+
227
+@pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
228
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
229
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
230
+@pytest.mark.parametrize("dtype", DTYPES)
231
+@pytest.mark.parametrize("seed", SEEDS)
232
@torch.inference_mode()
250
-def run_multi_query_kv_attention(
233
+def test_multi_query_kv_attention(
234
num_seqs: int,
252
- num_heads: int,
235
+ num_heads: Tuple[int, int],
236
head_size: int,
237
dtype: torch.dtype,
238
+ seed: int,
239
) -> None:
240
+ random.seed(seed)
241
+ torch.random.manual_seed(seed)
242
+ torch.cuda.manual_seed(seed)
243
+
244
seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
245
num_tokens = sum(seq_lens)
246
247
scale = float(1.0 / (head_size**0.5))
248
+ num_query_heads, num_kv_heads = num_heads
249
qkv = torch.empty(num_tokens,
261
- 3,
262
- num_heads,
250
+ num_query_heads + 2 * num_kv_heads,
251
head_size,
252
dtype=dtype,
265
- device='cuda')
266
- qkv.uniform_(-1e-3, 1e-3)
267
- query, key, value = qkv.unbind(dim=1)
268
-
253
+ device="cuda")
254
+ qkv.uniform_(-scale, scale)
255
+ query, key, value = qkv.split(
256
+ [num_query_heads, num_kv_heads, num_kv_heads], dim=1)
257
+
258
+ num_queries_per_kv = num_query_heads // num_kv_heads
259
+ if num_queries_per_kv > 1:
260
+ # Handle MQA and GQA
261
+ key = torch.repeat_interleave(key, num_queries_per_kv, dim=1)
262
+ value = torch.repeat_interleave(value, num_queries_per_kv, dim=1)
263
attn_bias = BlockDiagonalCausalMask.from_seqlens(seq_lens)
264
output = xops.memory_efficient_attention_forward(
265
query.unsqueeze(0),
@@ -285,40 +279,7 @@ def run_multi_query_kv_attention(
279
query,
280
key,
281
value,
282
+ scale,
283
dtype,
284
)
285
assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
291
-
292
-
293
-def test_single_query_cached_kv_attention() -> None:
294
- torch.random.manual_seed(TEST_SEED)
295
- torch.cuda.manual_seed(TEST_SEED)
296
- for dtype in [torch.half, torch.bfloat16, torch.float]:
297
- for block_size in [8, 16, 32]:
298
- for head_size in [64, 80, 96, 112, 128, 256]:
299
- print(f'Testing single_query_cached_kv_attention with '
300
- f'dtype={dtype}, block_size={block_size}, '
301
- f'head_size={head_size}')
302
- run_single_query_cached_kv_attention(
303
- num_tokens=37,
304
- num_heads=3,
305
- head_size=head_size,
306
- block_size=block_size,
307
- num_blocks=1024,
308
- dtype=dtype,
309
- )
310
-
311
-
312
-def test_multi_query_kv_attention() -> None:
313
- torch.random.manual_seed(TEST_SEED)
314
- torch.cuda.manual_seed(TEST_SEED)
315
- for dtype in [torch.half, torch.bfloat16, torch.float]:
316
- for head_size in [64, 80, 96, 112, 128, 256]:
317
- print(f'Testing multi_query_kv_attention with dtype={dtype}, '
318
- f'head_size={head_size}')
319
- run_multi_query_kv_attention(
320
- num_seqs=5,
321
- num_heads=3,
322
- head_size=head_size,
323
- dtype=dtype,
324
- )
tests/kernels/test_cache.py
+80
-130
@@ -1,12 +1,32 @@
1
import random
2
3
+import pytest
4
import torch
5
6
from vllm import cache_ops
7
7
-
8
+DTYPES = [torch.half, torch.bfloat16, torch.float]
9
+NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
10
+NUM_LAYERS = [5] # Arbitrary values for testing
11
+NUM_HEADS = [8] # Arbitrary values for testing
12
+HEAD_SIZES = [64, 80, 96, 112, 128, 256]
13
+BLOCK_SIZES = [8, 16, 32]
14
+NUM_BLOCKS = [1024] # Arbitrary values for testing
15
+NUM_MAPPINGS = [32, 256] # Arbitrary values for testing
16
+SEEDS = [0]
17
+
18
+
19
+@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
20
+@pytest.mark.parametrize("num_layers", NUM_LAYERS)
21
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
22
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
23
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
24
+@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
25
+@pytest.mark.parametrize("dtype", DTYPES)
26
+@pytest.mark.parametrize("seed", SEEDS)
27
@torch.inference_mode()
9
-def run_copy_blocks(
28
+def test_copy_blocks(
29
+ kv_cache_factory,
30
num_mappings: int,
31
num_layers: int,
32
num_heads: int,
@@ -14,48 +34,43 @@ def run_copy_blocks(
34
block_size: int,
35
num_blocks: int,
36
dtype: torch.dtype,
37
+ seed: int,
38
) -> None:
18
- # Generate random block mappings.
39
+ random.seed(seed)
40
+ torch.random.manual_seed(seed)
41
+ torch.cuda.manual_seed(seed)
42
+
43
+ # Generate random block mappings where each source block is mapped to two
44
+ # destination blocks.
45
+ assert 2 * num_mappings <= num_blocks
46
src_blocks = random.sample(range(num_blocks), num_mappings)
47
remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
21
- dst_blocks = random.sample(remainig_blocks, num_mappings)
22
- block_mapping = {src: [dst] for src, dst in zip(src_blocks, dst_blocks)}
23
-
24
- # Create the KV cache.
25
- x = 16 // torch.tensor([], dtype=dtype).element_size()
26
- key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
27
- key_caches = []
28
- for _ in range(num_layers):
29
- key_cache = torch.randn(size=key_cache_shape,
30
- dtype=dtype,
31
- device='cuda')
32
- key_caches.append(key_cache)
33
- cloned_key_caches = []
34
- for key_cache in key_caches:
35
- cloned_key_caches.append(key_cache.clone())
36
-
37
- value_cache_shape = (num_blocks, num_heads, head_size, block_size)
38
- value_caches = []
39
- for _ in range(num_layers):
40
- value_cache = torch.randn(size=value_cache_shape,
41
- dtype=dtype,
42
- device='cuda')
43
- value_caches.append(value_cache)
44
- cloned_value_caches = []
45
- for value_cache in value_caches:
46
- cloned_value_caches.append(value_cache.clone())
48
+ dst_blocks = random.sample(remainig_blocks, 2 * num_mappings)
49
+ block_mapping = {}
50
+ for i in range(num_mappings):
51
+ src = src_blocks[i]
52
+ dst1 = dst_blocks[2 * i]
53
+ dst2 = dst_blocks[2 * i + 1]
54
+ block_mapping[src] = [dst1, dst2]
55
+
56
+ # Create the KV caches.
57
+ key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
58
+ num_layers, num_heads,
59
+ head_size, dtype, seed)
60
+
61
+ # Clone the KV caches.
62
+ cloned_key_caches = [key_cache.clone() for key_cache in key_caches]
63
+ cloned_value_caches = [value_cache.clone() for value_cache in value_caches]
64
65
# Call the copy blocks kernel.
66
cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
67
51
- # Reference implementation.
68
+ # Run the reference implementation.
69
for src, dsts in block_mapping.items():
70
for dst in dsts:
54
- for key_cache, cloned_key_cache in zip(key_caches,
55
- cloned_key_caches):
71
+ for cloned_key_cache in cloned_key_caches:
72
cloned_key_cache[dst] = cloned_key_cache[src]
57
- for value_cache, cloned_value_cache in zip(value_caches,
58
- cloned_value_caches):
73
+ for cloned_value_cache in cloned_value_caches:
74
cloned_value_cache[dst] = cloned_value_cache[src]
75
76
# Compare the results.
@@ -66,15 +81,29 @@ def run_copy_blocks(
81
assert torch.allclose(value_cache, cloned_value_cache)
82
83
84
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
85
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
86
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
87
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
88
+@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
89
+@pytest.mark.parametrize("dtype", DTYPES)
90
+@pytest.mark.parametrize("seed", SEEDS)
91
@torch.inference_mode()
70
-def run_reshape_and_cache(
92
+def test_reshape_and_cache(
93
+ kv_cache_factory,
94
num_tokens: int,
95
num_heads: int,
96
head_size: int,
97
block_size: int,
98
num_blocks: int,
99
dtype: torch.dtype,
100
+ seed: int,
101
) -> None:
102
+ random.seed(seed)
103
+ torch.random.manual_seed(seed)
104
+ torch.cuda.manual_seed(seed)
105
+
106
+ # Create a random slot mapping.
107
num_slots = block_size * num_blocks
108
slot_mapping = random.sample(range(num_slots), num_tokens)
109
slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
@@ -87,110 +116,31 @@ def run_reshape_and_cache(
116
device='cuda')
117
_, key, value = qkv.unbind(dim=1)
118
90
- x = 16 // torch.tensor([], dtype=dtype).element_size()
91
- key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
92
- key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
93
- cloned_key_cache = key_cache.clone()
119
+ # Create the KV caches.
120
+ key_caches, value_caches = kv_cache_factory(num_blocks, block_size, 1,
121
+ num_heads, head_size, dtype,
122
+ seed)
123
+ key_cache, value_cache = key_caches[0], value_caches[0]
124
95
- value_cache_shape = (num_blocks, num_heads, head_size, block_size)
96
- value_cache = torch.randn(size=value_cache_shape,
97
- dtype=dtype,
98
- device='cuda')
125
+ # Clone the KV caches.
126
+ cloned_key_cache = key_cache.clone()
127
cloned_value_cache = value_cache.clone()
128
129
+ # Call the reshape_and_cache kernel.
130
cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
131
slot_mapping)
132
133
+ # Run the reference implementation.
134
+ reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
135
+ block_indicies = torch.div(slot_mapping, block_size, rounding_mode='floor')
136
+ block_indicies = block_indicies.cpu().tolist()
137
+ block_offsets = slot_mapping % block_size
138
+ block_offsets = block_offsets.cpu().tolist()
139
for i in range(num_tokens):
105
- reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
106
- block_idx = torch.div(slot_mapping[i],
107
- block_size,
108
- rounding_mode='floor')
109
- block_offset = slot_mapping[i] % block_size
140
+ block_idx = block_indicies[i]
141
+ block_offset = block_offsets[i]
142
cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
143
cloned_value_cache[block_idx, :, :, block_offset] = value[i]
144
145
assert torch.allclose(key_cache, cloned_key_cache)
146
assert torch.allclose(value_cache, cloned_value_cache)
115
-
116
-
117
-@torch.inference_mode()
118
-def run_gather_cached_kv(
119
- num_tokens: int,
120
- num_heads: int,
121
- head_size: int,
122
- block_size: int,
123
- num_blocks: int,
124
- dtype: torch.dtype,
125
-) -> None:
126
- num_slots = block_size * num_blocks
127
- slot_mapping = random.sample(range(num_slots), num_tokens)
128
- slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
129
-
130
- qkv = torch.randn(num_tokens,
131
- 3,
132
- num_heads,
133
- head_size,
134
- dtype=dtype,
135
- device='cuda')
136
- _, key, value = qkv.unbind(dim=1)
137
-
138
- qkv_clone = qkv.clone()
139
- _, cloned_key, cloned_value = qkv_clone.unbind(dim=1)
140
-
141
- x = 16 // torch.tensor([], dtype=dtype).element_size()
142
- key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
143
- key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
144
-
145
- value_cache_shape = (num_blocks, num_heads, head_size, block_size)
146
- value_cache = torch.randn(size=value_cache_shape,
147
- dtype=dtype,
148
- device='cuda')
149
-
150
- cache_ops.gather_cached_kv(key, value, key_cache, value_cache,
151
- slot_mapping)
152
-
153
- # Reference implementation.
154
- for i in range(num_tokens):
155
- reshaped_key = cloned_key.reshape(num_tokens, num_heads,
156
- head_size // x, x)
157
- block_idx = torch.div(slot_mapping[i],
158
- block_size,
159
- rounding_mode='floor')
160
- block_offset = slot_mapping[i] % block_size
161
- reshaped_key[i] = key_cache[block_idx, :, :, block_offset, :]
162
- cloned_value[i] = value_cache[block_idx, :, :, block_offset]
163
-
164
- assert torch.allclose(key, cloned_key)
165
- assert torch.allclose(value, cloned_value)
166
-
167
-
168
-def test_copy_blocks() -> None:
169
- for dtype in [torch.half, torch.bfloat16, torch.float]:
170
- run_copy_blocks(num_mappings=23,
171
- num_layers=7,
172
- num_heads=17,
173
- head_size=16,
174
- block_size=8,
175
- num_blocks=1024,
176
- dtype=dtype)
177
-
178
-
179
-def test_reshape_and_cache() -> None:
180
- for dtype in [torch.half, torch.bfloat16, torch.float]:
181
- run_reshape_and_cache(num_tokens=3,
182
- num_heads=2,
183
- head_size=16,
184
- block_size=8,
185
- num_blocks=2,
186
- dtype=dtype)
187
-
188
-
189
-def test_gather_cached_kv() -> None:
190
- for dtype in [torch.half, torch.bfloat16, torch.float]:
191
- run_gather_cached_kv(num_tokens=3,
192
- num_heads=2,
193
- head_size=16,
194
- block_size=8,
195
- num_blocks=2,
196
- dtype=dtype)
tests/kernels/test_layernorm.py
+24
-22
@@ -1,35 +1,50 @@
1
+import pytest
2
import torch
3
import torch.nn as nn
4
5
from vllm import layernorm_ops
6
7
+DTYPES = [torch.half, torch.bfloat16, torch.float]
8
+HIDDEN_SIZES = [67, 768, 2048, 5120, 8192] # Arbitrary values for testing
9
+NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
10
+SEEDS = [0]
11
+
12
13
class RefRMSNorm(nn.Module):
14
15
def __init__(self, hidden_size, eps=1e-6):
16
super().__init__()
17
weight = torch.empty(hidden_size)
12
- weight.uniform_(-1e-3, 1e-3)
18
+ weight.normal_(mean=1.0, std=0.1)
19
self.weight = nn.Parameter(weight)
20
self.variance_epsilon = eps
21
22
def forward(self, hidden_states):
17
- variance = hidden_states.to(torch.float32).pow(2).mean(-1,
18
- keepdim=True)
23
+ input_dtype = hidden_states.dtype
24
+ hidden_states = hidden_states.to(torch.float32)
25
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
26
hidden_states = hidden_states * torch.rsqrt(variance +
27
self.variance_epsilon)
21
- if self.weight.dtype in [torch.half, torch.float16, torch.bfloat16]:
22
- hidden_states = hidden_states.to(self.weight.dtype)
23
- return self.weight * hidden_states
28
+ return self.weight * hidden_states.to(input_dtype)
29
30
31
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
32
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
33
+@pytest.mark.parametrize("dtype", DTYPES)
34
+@pytest.mark.parametrize("seed", SEEDS)
35
@torch.inference_mode()
27
-def run_rms_norm(
36
+def test_rms_norm(
37
num_tokens: int,
38
hidden_size: int,
39
dtype: torch.dtype,
40
+ seed: int,
41
) -> None:
32
- x = torch.randn(num_tokens, hidden_size, dtype=dtype, device='cuda')
42
+ torch.random.manual_seed(seed)
43
+ torch.cuda.manual_seed(seed)
44
+
45
+ scale = float(hidden_size**-0.5)
46
+ x = torch.empty(num_tokens, hidden_size, dtype=dtype, device="cuda")
47
+ x.uniform_(-scale, scale)
48
ref = RefRMSNorm(hidden_size).to(dtype).cuda()
49
50
out = torch.empty_like(x)
@@ -40,17 +55,4 @@ def run_rms_norm(
55
ref.variance_epsilon,
56
)
57
ref_out = ref(x)
43
- assert torch.allclose(out, ref_out, atol=1e-3, rtol=1e-5)
44
-
45
-
46
-def test_rms_norm() -> None:
47
- for dtype in [torch.half, torch.bfloat16, torch.float]:
48
- for num_tokens in [7, 128, 2048]:
49
- for hidden_size in [13, 64, 1024, 5120]:
50
- print(f'Testing RMS kernel with dtype={dtype}, num_tokens='
51
- f'{num_tokens}, hidden_size={hidden_size}')
52
- run_rms_norm(
53
- num_tokens=num_tokens,
54
- hidden_size=hidden_size,
55
- dtype=dtype,
56
- )
58
+ assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-5)
tests/kernels/test_pos_encoding.py
+27
-21
@@ -1,11 +1,19 @@
1
-from typing import Tuple
1
+from typing import Optional, Tuple
2
3
+import pytest
4
import torch
5
import torch.nn as nn
6
import torch.nn.functional as F
7
8
from vllm import pos_encoding_ops
9
10
+DTYPES = [torch.half, torch.bfloat16, torch.float]
11
+HEAD_SIZES = [64, 80, 96, 112, 128, 256]
12
+ROTARY_DIMS = [None, 32] # None means rotary dim == head size
13
+NUM_HEADS = [7, 12, 40, 52] # Arbitrary values for testing
14
+NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
15
+SEEDS = [0]
16
+
17
18
def rotate_half(x: torch.Tensor) -> torch.Tensor:
19
x1 = x[..., :x.shape[-1] // 2]
@@ -74,16 +82,28 @@ class RefRotaryEmbeddingNeox(nn.Module):
82
return query, key
83
84
85
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
86
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
87
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
88
+@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
89
+@pytest.mark.parametrize("dtype", DTYPES)
90
+@pytest.mark.parametrize("seed", SEEDS)
91
@torch.inference_mode()
78
-def run_rotary_embedding_neox(
92
+def test_rotary_embedding_neox(
93
num_tokens: int,
94
num_heads: int,
95
head_size: int,
82
- max_position: int,
83
- rotary_dim: int,
96
+ rotary_dim: Optional[int],
97
dtype: torch.dtype,
98
+ seed: int,
99
+ max_position: int = 8192,
100
base: int = 10000,
101
) -> None:
102
+ if rotary_dim is None:
103
+ rotary_dim = head_size
104
+ torch.random.manual_seed(seed)
105
+ torch.cuda.manual_seed(seed)
106
+
107
positions = torch.randint(0, max_position, (num_tokens, ), device='cuda')
108
query = torch.randn(num_tokens,
109
num_heads * head_size,
@@ -97,7 +117,7 @@ def run_rotary_embedding_neox(
117
# Create the rotary embedding.
118
inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
119
t = torch.arange(max_position).float()
100
- freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
120
+ freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
121
cos = freqs.cos()
122
sin = freqs.sin()
123
cos_sin_cache = torch.cat((cos, sin), dim=-1)
@@ -129,19 +149,5 @@ def run_rotary_embedding_neox(
149
ref_key = ref_key.view(num_tokens, num_heads * head_size)
150
151
# Compare the results.
132
- assert torch.allclose(out_query, ref_query, atol=1e-3, rtol=1e-5)
133
- assert torch.allclose(out_key, ref_key, atol=1e-3, rtol=1e-5)
134
-
135
-
136
-def test_rotary_embedding_neox() -> None:
137
- for dtype in [torch.half, torch.bfloat16, torch.float]:
138
- for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
139
- print(f'Running tests for head_size={head_size} and dtype={dtype}')
140
- run_rotary_embedding_neox(
141
- num_tokens=2145,
142
- num_heads=5,
143
- head_size=head_size,
144
- max_position=8192,
145
- rotary_dim=head_size,
146
- dtype=dtype,
147
- )
152
+ assert torch.allclose(out_query, ref_query, atol=1e-5, rtol=1e-5)
153
+ assert torch.allclose(out_key, ref_key, atol=1e-5, rtol=1e-5)