@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fbd80ad409
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Clean up kernel unit tests (#938)

Woosuk Kwon committed Sep 6, 2023 at 08:57 UTC fbd80ad4092c4bc48ce672f0435c1d1362aee052
6 files changed +364 -399
tests/kernels/conftest.py new
+43
@@ -0,0 +1,43 @@
1 +from typing import List, Tuple
2 +
3 +import pytest
4 +import torch
5 +
6 +
7 +def create_kv_caches(
8 + num_blocks: int,
9 + block_size: int,
10 + num_layers: int,
11 + num_heads: int,
12 + head_size: int,
13 + dtype: torch.dtype,
14 + seed: int,
15 +) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
16 + torch.random.manual_seed(seed)
17 + torch.cuda.manual_seed(seed)
18 +
19 + scale = head_size**-0.5
20 + x = 16 // torch.tensor([], dtype=dtype).element_size()
21 + key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
22 + key_caches = []
23 + for _ in range(num_layers):
24 + key_cache = torch.empty(size=key_cache_shape,
25 + dtype=dtype,
26 + device='cuda')
27 + key_cache.uniform_(-scale, scale)
28 + key_caches.append(key_cache)
29 +
30 + value_cache_shape = (num_blocks, num_heads, head_size, block_size)
31 + value_caches = []
32 + for _ in range(num_layers):
33 + value_cache = torch.empty(size=value_cache_shape,
34 + dtype=dtype,
35 + device='cuda')
36 + value_cache.uniform_(-scale, scale)
37 + value_caches.append(value_cache)
38 + return key_caches, value_caches
39 +
40 +
41 +@pytest.fixture()
42 +def kv_cache_factory():
43 + return create_kv_caches
tests/kernels/test_activation.py
+31 -28
@@ -1,20 +1,34 @@
1 +import pytest
2 import torch
3 import torch.nn.functional as F
4 from transformers.activations import get_activation
5 +
6 from vllm import activation_ops
7
8 +DTYPES = [torch.half, torch.bfloat16, torch.float]
9 +NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
10 +D = [512, 4096, 5120, 13824] # Arbitrary values for testing
11 +SEEDS = [0]
12 +
13
14 def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
15 x1, x2 = x.chunk(chunks=2, dim=1)
16 return F.silu(x1) * x2
17
18
19 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
20 +@pytest.mark.parametrize("d", D)
21 +@pytest.mark.parametrize("dtype", DTYPES)
22 +@pytest.mark.parametrize("seed", SEEDS)
23 @torch.inference_mode()
13 -def run_silu_and_mul(
24 +def test_silu_and_mul(
25 num_tokens: int,
26 d: int,
27 dtype: torch.dtype,
28 + seed: int,
29 ) -> None:
30 + torch.random.manual_seed(seed)
31 + torch.cuda.manual_seed(seed)
32 x = torch.randn(num_tokens, 2 * d, dtype=dtype, device='cuda')
33 out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
34 activation_ops.silu_and_mul(out, x)
@@ -22,20 +36,19 @@ def run_silu_and_mul(
36 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
37
38
25 -def test_silu_and_mul() -> None:
26 - for dtype in [torch.half, torch.bfloat16, torch.float]:
27 - for num_tokens in [7, 83, 2048]:
28 - for d in [512, 4096, 5120, 13824]:
29 - print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
30 - run_silu_and_mul(num_tokens, d, dtype)
31 -
32 -
39 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
40 +@pytest.mark.parametrize("d", D)
41 +@pytest.mark.parametrize("dtype", DTYPES)
42 +@pytest.mark.parametrize("seed", SEEDS)
43 @torch.inference_mode()
34 -def run_gelu_new(
44 +def test_gelu_new(
45 num_tokens: int,
46 d: int,
47 dtype: torch.dtype,
48 + seed: int,
49 ) -> None:
50 + torch.random.manual_seed(seed)
51 + torch.cuda.manual_seed(seed)
52 x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
53 out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
54 activation_ops.gelu_new(out, x)
@@ -43,30 +56,20 @@ def run_gelu_new(
56 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
57
58
46 -def test_gelu_new() -> None:
47 - for dtype in [torch.half, torch.bfloat16, torch.float]:
48 - for num_tokens in [7, 83, 2048]:
49 - for d in [512, 4096, 5120, 13824]:
50 - print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
51 - run_gelu_new(num_tokens, d, dtype)
52 -
53 -
54 -@torch.inference_mode()
55 -def run_gelu_fast(
59 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
60 +@pytest.mark.parametrize("d", D)
61 +@pytest.mark.parametrize("dtype", DTYPES)
62 +@pytest.mark.parametrize("seed", SEEDS)
63 +def test_gelu_fast(
64 num_tokens: int,
65 d: int,
66 dtype: torch.dtype,
67 + seed: int,
68 ) -> None:
69 + torch.random.manual_seed(seed)
70 + torch.cuda.manual_seed(seed)
71 x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
72 out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
73 activation_ops.gelu_fast(out, x)
74 ref_out = get_activation("gelu_fast")(x)
75 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
65 -
66 -
67 -def test_gelu_fast() -> None:
68 - for dtype in [torch.half, torch.bfloat16, torch.float]:
69 - for num_tokens in [7, 83, 2048]:
70 - for d in [512, 4096, 5120, 13824]:
71 - print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
72 - run_gelu_fast(num_tokens, d, dtype)
tests/kernels/test_attention.py
+159 -198
@@ -1,14 +1,24 @@
1 import random
2 -from typing import List, Optional
2 +from typing import List, Optional, Tuple
3
4 +import pytest
5 import torch
6 from xformers import ops as xops
7 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9 from vllm import attention_ops
10
10 -MAX_SEQ_LEN = 4096
11 -TEST_SEED = 0
11 +MAX_SEQ_LEN = 8192
12 +NUM_BLOCKS = 128 # Arbitrary values for testing
13 +
14 +DTYPES = [torch.half, torch.bfloat16, torch.float]
15 +NUM_GEN_SEQS = [7] # Arbitrary values for testing
16 +NUM_PREFILL_SEQS = [1, 3, 7] # Arbitrary values for testing
17 +NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
18 +HEAD_SIZES = [64, 80, 96, 112, 128, 256]
19 +BLOCK_SIZES = [8, 16, 32]
20 +USE_ALIBI = [False] # TODO(woosuk): Add USE_ALIBI=True
21 +SEEDS = [0]
22
23
24 def ref_masked_attention(
@@ -18,29 +28,34 @@ def ref_masked_attention(
28 scale: float,
29 attn_mask: Optional[torch.Tensor] = None,
30 ) -> torch.Tensor:
21 - query = query * scale
22 - attn = torch.einsum('qhd,khd->hqk', query, key)
31 + attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
32 if attn_mask is not None:
24 - attn = attn + attn_mask
25 - attn = torch.softmax(attn, dim=-1)
26 - out = torch.einsum('hqk,khd->qhd', attn, value)
33 + attn_weights = attn_weights + attn_mask.float()
34 + attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
35 + out = torch.einsum("hqk,khd->qhd", attn_weights, value)
36 return out
37
38
39 def ref_single_query_cached_kv_attention(
40 output: torch.Tensor,
41 query: torch.Tensor,
42 + num_queries_per_kv: int,
43 key_cache: torch.Tensor,
44 value_cache: torch.Tensor,
45 block_tables: torch.Tensor,
46 context_lens: torch.Tensor,
47 + scale: float,
48 + alibi_slopes: Optional[torch.Tensor],
49 ) -> None:
38 - num_heads = value_cache.shape[1]
50 + num_query_heads = query.shape[1]
51 + num_kv_heads = value_cache.shape[1]
52 head_size = value_cache.shape[2]
53 block_size = value_cache.shape[3]
54 + num_seqs = query.shape[0]
55
42 - num_input_tokens = query.shape[0]
43 - for i in range(num_input_tokens):
56 + block_tables = block_tables.cpu().tolist()
57 + context_lens = context_lens.cpu().tolist()
58 + for i in range(num_seqs):
59 q = query[i].unsqueeze(0)
60 block_table = block_tables[i]
61 context_len = int(context_lens[i])
@@ -52,170 +67,96 @@ def ref_single_query_cached_kv_attention(
67 block_offset = j % block_size
68
69 k = key_cache[block_number, :, :, block_offset, :]
55 - k = k.reshape(num_heads, head_size)
70 + k = k.reshape(num_kv_heads, head_size)
71 keys.append(k)
72
73 v = value_cache[block_number, :, :, block_offset]
74 values.append(v)
75 keys = torch.stack(keys, dim=0)
76 values = torch.stack(values, dim=0)
62 -
63 - scale = 1.0 / (head_size**0.5)
64 - out = ref_masked_attention(q, keys, values, scale)
65 - out = out.view(num_heads, head_size)
77 + if num_queries_per_kv > 1:
78 + # Handle MQA and GQA
79 + keys = torch.repeat_interleave(keys, num_queries_per_kv, dim=1)
80 + values = torch.repeat_interleave(values, num_queries_per_kv, dim=1)
81 +
82 + alibi_bias = None
83 + if alibi_slopes is not None:
84 + # Create the ALiBi bias used in the paged attention kernel.
85 + position_ids = torch.arange(context_len, device="cuda").int()
86 + alibi_bias = (context_len - position_ids).float()
87 + alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
88 + 1, 1, -1)
89 +
90 + out = ref_masked_attention(q, keys, values, scale, alibi_bias)
91 + out = out.view(num_query_heads, head_size)
92 output[i].copy_(out, non_blocking=True)
93
94
69 -def ref_multi_query_kv_attention(
70 - cu_seq_lens: List[int],
71 - query: torch.Tensor,
72 - key: torch.Tensor,
73 - value: torch.Tensor,
74 - dtype: torch.dtype,
75 -) -> torch.Tensor:
76 - head_size = query.shape[-1]
77 - scale = 1.0 / (head_size**0.5)
78 -
79 - num_seqs = len(cu_seq_lens) - 1
80 - ref_outputs = []
81 - for i in range(num_seqs):
82 - start_idx = cu_seq_lens[i]
83 - end_idx = cu_seq_lens[i + 1]
84 - seq_len = end_idx - start_idx
85 -
86 - # Create attention mask.
87 - attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
88 - diagonal=1)
89 - attn_mask = attn_mask * torch.finfo(dtype).min
90 - attn_mask = attn_mask.to(dtype=dtype, device='cuda')
91 -
92 - ref_output = ref_masked_attention(
93 - query[start_idx:end_idx],
94 - key[start_idx:end_idx],
95 - value[start_idx:end_idx],
96 - scale,
97 - attn_mask=attn_mask,
98 - )
99 - ref_outputs.append(ref_output)
100 - ref_output = torch.cat(ref_outputs, dim=0)
101 - return ref_output
102 -
103 -
104 -def ref_multi_query_cached_kv_attention(
105 - cu_query_lens: List[int],
106 - query: torch.Tensor,
107 - key_cache: torch.Tensor,
108 - value_cache: torch.Tensor,
109 - block_tables: torch.Tensor,
110 - context_lens: torch.Tensor,
111 - dtype: torch.dtype,
112 -) -> torch.Tensor:
113 - num_heads = value_cache.shape[1]
114 - head_size = value_cache.shape[2]
115 - block_size = value_cache.shape[3]
116 - scale = 1.0 / (head_size**0.5)
117 -
118 - num_queries = len(cu_query_lens) - 1
119 - ref_outputs = []
120 - for i in range(num_queries):
121 - start_idx = cu_query_lens[i]
122 - end_idx = cu_query_lens[i + 1]
123 - query_len = end_idx - start_idx
124 - context_len = int(context_lens[i])
125 - block_table = block_tables[i]
126 -
127 - # Create attention mask
128 - attn_mask = torch.triu(torch.ones(query_len, context_len),
129 - diagonal=context_len - query_len + 1) * -1e5
130 - attn_mask = attn_mask.to(dtype=dtype, device='cuda')
131 -
132 - keys = []
133 - values = []
134 - for j in range(context_len):
135 - block_number = int(block_table[j // block_size])
136 - block_offset = j % block_size
137 -
138 - k = key_cache[block_number, :, :, block_offset, :]
139 - k = k.reshape(num_heads, head_size)
140 - keys.append(k)
141 -
142 - v = value_cache[block_number, :, :, block_offset]
143 - values.append(v)
144 - keys = torch.stack(keys, dim=0)
145 - values = torch.stack(values, dim=0)
146 -
147 - ref_output = ref_masked_attention(
148 - query[start_idx:end_idx],
149 - keys,
150 - values,
151 - scale,
152 - attn_mask=attn_mask,
153 - )
154 - ref_outputs.append(ref_output)
155 - ref_output = torch.cat(ref_outputs, dim=0)
156 - return ref_output
157 -
158 -
95 +@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
96 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
97 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
98 +@pytest.mark.parametrize("use_alibi", USE_ALIBI)
99 +@pytest.mark.parametrize("block_size", BLOCK_SIZES)
100 +@pytest.mark.parametrize("dtype", DTYPES)
101 +@pytest.mark.parametrize("seed", SEEDS)
102 @torch.inference_mode()
160 -def run_single_query_cached_kv_attention(
161 - num_tokens: int,
162 - num_heads: int,
103 +def test_single_query_cached_kv_attention(
104 + kv_cache_factory,
105 + num_seqs: int,
106 + num_heads: Tuple[int, int],
107 head_size: int,
108 + use_alibi: bool,
109 block_size: int,
165 - num_blocks: int,
110 dtype: torch.dtype,
167 - num_kv_heads: int = None,
111 + seed: int,
112 ) -> None:
169 - qkv = torch.empty(num_tokens,
170 - 3,
171 - num_heads,
172 - head_size,
173 - dtype=dtype,
174 - device='cuda')
175 - qkv.uniform_(-1e-3, 1e-3)
176 - query, _, _ = qkv.unbind(dim=1)
177 -
178 - x = 16 // torch.tensor([], dtype=dtype).element_size()
179 - key_block_shape = (num_heads, head_size // x, block_size, x)
180 - key_cache = torch.empty(size=(num_blocks, *key_block_shape),
181 - dtype=dtype,
182 - device='cuda')
183 - key_cache.uniform_(-1e-3, 1e-3)
184 - value_block_shape = (num_heads, head_size, block_size)
185 - value_cache = torch.empty(size=(num_blocks, *value_block_shape),
186 - dtype=dtype,
187 - device='cuda')
188 - value_cache.uniform_(-1e-3, 1e-3)
189 -
190 - context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
113 + random.seed(seed)
114 + torch.random.manual_seed(seed)
115 + torch.cuda.manual_seed(seed)
116 +
117 + scale = float(1.0 / (head_size**0.5))
118 + num_query_heads, num_kv_heads = num_heads
119 + query = torch.empty(num_seqs,
120 + num_query_heads,
121 + head_size,
122 + dtype=dtype,
123 + device="cuda")
124 + query.uniform_(-scale, scale)
125 +
126 + assert num_query_heads % num_kv_heads == 0
127 + num_queries_per_kv = num_query_heads // num_kv_heads
128 + head_mapping = torch.repeat_interleave(
129 + torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
130 + num_queries_per_kv)
131 + alibi_slopes = None
132 + if use_alibi:
133 + alibi_slopes = torch.randn(num_query_heads,
134 + dtype=torch.float,
135 + device="cuda")
136 +
137 + context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
138 max_context_len = max(context_lens)
192 - context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
139 + context_lens = torch.tensor(context_lens, dtype=torch.int, device="cuda")
140
141 + # Create the block tables.
142 max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
143 block_tables = []
196 - for _ in range(num_tokens):
144 + for _ in range(num_seqs):
145 block_table = [
198 - random.randint(0, num_blocks - 1)
146 + random.randint(0, NUM_BLOCKS - 1)
147 for _ in range(max_num_blocks_per_seq)
148 ]
149 block_tables.append(block_table)
202 - block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
203 - head_mapping = torch.arange(num_heads, dtype=torch.int32, device="cuda")
150 + block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
151
205 - scale = float(1.0 / (head_size**0.5))
206 -
207 - num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
208 - assert num_heads % num_kv_heads == 0
209 - num_queries_per_kv = num_heads // num_kv_heads
210 - head_mapping = torch.repeat_interleave(
211 - torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
212 - num_queries_per_kv)
152 + # Create the KV caches.
153 + key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
154 + num_kv_heads, head_size, dtype,
155 + seed)
156 + key_cache, value_cache = key_caches[0], value_caches[0]
157
214 - output = torch.empty(num_tokens,
215 - num_heads,
216 - head_size,
217 - dtype=dtype,
218 - device='cuda')
158 + # Call the paged attention kernel.
159 + output = torch.empty_like(query)
160 attention_ops.single_query_cached_kv_attention(
161 output,
162 query,
@@ -227,45 +168,98 @@ def run_single_query_cached_kv_attention(
168 context_lens,
169 block_size,
170 max_context_len,
230 - None, # ALiBi slopes.
171 + alibi_slopes,
172 )
173
174 + # Run the reference implementation.
175 ref_output = torch.empty_like(query)
176 ref_single_query_cached_kv_attention(
177 ref_output,
178 query,
179 + num_queries_per_kv,
180 key_cache,
181 value_cache,
182 block_tables,
183 context_lens,
184 + scale,
185 + alibi_slopes,
186 )
242 - # NOTE(woosuk): Due to the difference in the data types the two
243 - # implementations use for attention softmax logits and accumulation,
244 - # there is a small difference in the final outputs.
245 - # We should use a relaxed tolerance for the test.
187 +
188 + # NOTE(woosuk): Due to the kernel-level differences in the two
189 + # implementations, there is a small numerical difference in the two
190 + # outputs. Thus, we use a relaxed tolerance for the test.
191 assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
192
193
194 +def ref_multi_query_kv_attention(
195 + cu_seq_lens: List[int],
196 + query: torch.Tensor,
197 + key: torch.Tensor,
198 + value: torch.Tensor,
199 + scale: float,
200 + dtype: torch.dtype,
201 +) -> torch.Tensor:
202 + num_seqs = len(cu_seq_lens) - 1
203 + ref_outputs = []
204 + for i in range(num_seqs):
205 + start_idx = cu_seq_lens[i]
206 + end_idx = cu_seq_lens[i + 1]
207 + seq_len = end_idx - start_idx
208 +
209 + # Create attention mask.
210 + attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
211 + diagonal=1)
212 + attn_mask = attn_mask * torch.finfo(dtype).min
213 + attn_mask = attn_mask.to(dtype=dtype, device="cuda")
214 +
215 + ref_output = ref_masked_attention(
216 + query[start_idx:end_idx],
217 + key[start_idx:end_idx],
218 + value[start_idx:end_idx],
219 + scale,
220 + attn_mask=attn_mask,
221 + )
222 + ref_outputs.append(ref_output)
223 + ref_output = torch.cat(ref_outputs, dim=0)
224 + return ref_output
225 +
226 +
227 +@pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
228 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
229 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
230 +@pytest.mark.parametrize("dtype", DTYPES)
231 +@pytest.mark.parametrize("seed", SEEDS)
232 @torch.inference_mode()
250 -def run_multi_query_kv_attention(
233 +def test_multi_query_kv_attention(
234 num_seqs: int,
252 - num_heads: int,
235 + num_heads: Tuple[int, int],
236 head_size: int,
237 dtype: torch.dtype,
238 + seed: int,
239 ) -> None:
240 + random.seed(seed)
241 + torch.random.manual_seed(seed)
242 + torch.cuda.manual_seed(seed)
243 +
244 seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
245 num_tokens = sum(seq_lens)
246
247 scale = float(1.0 / (head_size**0.5))
248 + num_query_heads, num_kv_heads = num_heads
249 qkv = torch.empty(num_tokens,
261 - 3,
262 - num_heads,
250 + num_query_heads + 2 * num_kv_heads,
251 head_size,
252 dtype=dtype,
265 - device='cuda')
266 - qkv.uniform_(-1e-3, 1e-3)
267 - query, key, value = qkv.unbind(dim=1)
268 -
253 + device="cuda")
254 + qkv.uniform_(-scale, scale)
255 + query, key, value = qkv.split(
256 + [num_query_heads, num_kv_heads, num_kv_heads], dim=1)
257 +
258 + num_queries_per_kv = num_query_heads // num_kv_heads
259 + if num_queries_per_kv > 1:
260 + # Handle MQA and GQA
261 + key = torch.repeat_interleave(key, num_queries_per_kv, dim=1)
262 + value = torch.repeat_interleave(value, num_queries_per_kv, dim=1)
263 attn_bias = BlockDiagonalCausalMask.from_seqlens(seq_lens)
264 output = xops.memory_efficient_attention_forward(
265 query.unsqueeze(0),
@@ -285,40 +279,7 @@ def run_multi_query_kv_attention(
279 query,
280 key,
281 value,
282 + scale,
283 dtype,
284 )
285 assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
291 -
292 -
293 -def test_single_query_cached_kv_attention() -> None:
294 - torch.random.manual_seed(TEST_SEED)
295 - torch.cuda.manual_seed(TEST_SEED)
296 - for dtype in [torch.half, torch.bfloat16, torch.float]:
297 - for block_size in [8, 16, 32]:
298 - for head_size in [64, 80, 96, 112, 128, 256]:
299 - print(f'Testing single_query_cached_kv_attention with '
300 - f'dtype={dtype}, block_size={block_size}, '
301 - f'head_size={head_size}')
302 - run_single_query_cached_kv_attention(
303 - num_tokens=37,
304 - num_heads=3,
305 - head_size=head_size,
306 - block_size=block_size,
307 - num_blocks=1024,
308 - dtype=dtype,
309 - )
310 -
311 -
312 -def test_multi_query_kv_attention() -> None:
313 - torch.random.manual_seed(TEST_SEED)
314 - torch.cuda.manual_seed(TEST_SEED)
315 - for dtype in [torch.half, torch.bfloat16, torch.float]:
316 - for head_size in [64, 80, 96, 112, 128, 256]:
317 - print(f'Testing multi_query_kv_attention with dtype={dtype}, '
318 - f'head_size={head_size}')
319 - run_multi_query_kv_attention(
320 - num_seqs=5,
321 - num_heads=3,
322 - head_size=head_size,
323 - dtype=dtype,
324 - )
tests/kernels/test_cache.py
+80 -130
@@ -1,12 +1,32 @@
1 import random
2
3 +import pytest
4 import torch
5
6 from vllm import cache_ops
7
7 -
8 +DTYPES = [torch.half, torch.bfloat16, torch.float]
9 +NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
10 +NUM_LAYERS = [5] # Arbitrary values for testing
11 +NUM_HEADS = [8] # Arbitrary values for testing
12 +HEAD_SIZES = [64, 80, 96, 112, 128, 256]
13 +BLOCK_SIZES = [8, 16, 32]
14 +NUM_BLOCKS = [1024] # Arbitrary values for testing
15 +NUM_MAPPINGS = [32, 256] # Arbitrary values for testing
16 +SEEDS = [0]
17 +
18 +
19 +@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
20 +@pytest.mark.parametrize("num_layers", NUM_LAYERS)
21 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
22 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
23 +@pytest.mark.parametrize("block_size", BLOCK_SIZES)
24 +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
25 +@pytest.mark.parametrize("dtype", DTYPES)
26 +@pytest.mark.parametrize("seed", SEEDS)
27 @torch.inference_mode()
9 -def run_copy_blocks(
28 +def test_copy_blocks(
29 + kv_cache_factory,
30 num_mappings: int,
31 num_layers: int,
32 num_heads: int,
@@ -14,48 +34,43 @@ def run_copy_blocks(
34 block_size: int,
35 num_blocks: int,
36 dtype: torch.dtype,
37 + seed: int,
38 ) -> None:
18 - # Generate random block mappings.
39 + random.seed(seed)
40 + torch.random.manual_seed(seed)
41 + torch.cuda.manual_seed(seed)
42 +
43 + # Generate random block mappings where each source block is mapped to two
44 + # destination blocks.
45 + assert 2 * num_mappings <= num_blocks
46 src_blocks = random.sample(range(num_blocks), num_mappings)
47 remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
21 - dst_blocks = random.sample(remainig_blocks, num_mappings)
22 - block_mapping = {src: [dst] for src, dst in zip(src_blocks, dst_blocks)}
23 -
24 - # Create the KV cache.
25 - x = 16 // torch.tensor([], dtype=dtype).element_size()
26 - key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
27 - key_caches = []
28 - for _ in range(num_layers):
29 - key_cache = torch.randn(size=key_cache_shape,
30 - dtype=dtype,
31 - device='cuda')
32 - key_caches.append(key_cache)
33 - cloned_key_caches = []
34 - for key_cache in key_caches:
35 - cloned_key_caches.append(key_cache.clone())
36 -
37 - value_cache_shape = (num_blocks, num_heads, head_size, block_size)
38 - value_caches = []
39 - for _ in range(num_layers):
40 - value_cache = torch.randn(size=value_cache_shape,
41 - dtype=dtype,
42 - device='cuda')
43 - value_caches.append(value_cache)
44 - cloned_value_caches = []
45 - for value_cache in value_caches:
46 - cloned_value_caches.append(value_cache.clone())
48 + dst_blocks = random.sample(remainig_blocks, 2 * num_mappings)
49 + block_mapping = {}
50 + for i in range(num_mappings):
51 + src = src_blocks[i]
52 + dst1 = dst_blocks[2 * i]
53 + dst2 = dst_blocks[2 * i + 1]
54 + block_mapping[src] = [dst1, dst2]
55 +
56 + # Create the KV caches.
57 + key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
58 + num_layers, num_heads,
59 + head_size, dtype, seed)
60 +
61 + # Clone the KV caches.
62 + cloned_key_caches = [key_cache.clone() for key_cache in key_caches]
63 + cloned_value_caches = [value_cache.clone() for value_cache in value_caches]
64
65 # Call the copy blocks kernel.
66 cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
67
51 - # Reference implementation.
68 + # Run the reference implementation.
69 for src, dsts in block_mapping.items():
70 for dst in dsts:
54 - for key_cache, cloned_key_cache in zip(key_caches,
55 - cloned_key_caches):
71 + for cloned_key_cache in cloned_key_caches:
72 cloned_key_cache[dst] = cloned_key_cache[src]
57 - for value_cache, cloned_value_cache in zip(value_caches,
58 - cloned_value_caches):
73 + for cloned_value_cache in cloned_value_caches:
74 cloned_value_cache[dst] = cloned_value_cache[src]
75
76 # Compare the results.
@@ -66,15 +81,29 @@ def run_copy_blocks(
81 assert torch.allclose(value_cache, cloned_value_cache)
82
83
84 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
85 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
86 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
87 +@pytest.mark.parametrize("block_size", BLOCK_SIZES)
88 +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
89 +@pytest.mark.parametrize("dtype", DTYPES)
90 +@pytest.mark.parametrize("seed", SEEDS)
91 @torch.inference_mode()
70 -def run_reshape_and_cache(
92 +def test_reshape_and_cache(
93 + kv_cache_factory,
94 num_tokens: int,
95 num_heads: int,
96 head_size: int,
97 block_size: int,
98 num_blocks: int,
99 dtype: torch.dtype,
100 + seed: int,
101 ) -> None:
102 + random.seed(seed)
103 + torch.random.manual_seed(seed)
104 + torch.cuda.manual_seed(seed)
105 +
106 + # Create a random slot mapping.
107 num_slots = block_size * num_blocks
108 slot_mapping = random.sample(range(num_slots), num_tokens)
109 slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
@@ -87,110 +116,31 @@ def run_reshape_and_cache(
116 device='cuda')
117 _, key, value = qkv.unbind(dim=1)
118
90 - x = 16 // torch.tensor([], dtype=dtype).element_size()
91 - key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
92 - key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
93 - cloned_key_cache = key_cache.clone()
119 + # Create the KV caches.
120 + key_caches, value_caches = kv_cache_factory(num_blocks, block_size, 1,
121 + num_heads, head_size, dtype,
122 + seed)
123 + key_cache, value_cache = key_caches[0], value_caches[0]
124
95 - value_cache_shape = (num_blocks, num_heads, head_size, block_size)
96 - value_cache = torch.randn(size=value_cache_shape,
97 - dtype=dtype,
98 - device='cuda')
125 + # Clone the KV caches.
126 + cloned_key_cache = key_cache.clone()
127 cloned_value_cache = value_cache.clone()
128
129 + # Call the reshape_and_cache kernel.
130 cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
131 slot_mapping)
132
133 + # Run the reference implementation.
134 + reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
135 + block_indicies = torch.div(slot_mapping, block_size, rounding_mode='floor')
136 + block_indicies = block_indicies.cpu().tolist()
137 + block_offsets = slot_mapping % block_size
138 + block_offsets = block_offsets.cpu().tolist()
139 for i in range(num_tokens):
105 - reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
106 - block_idx = torch.div(slot_mapping[i],
107 - block_size,
108 - rounding_mode='floor')
109 - block_offset = slot_mapping[i] % block_size
140 + block_idx = block_indicies[i]
141 + block_offset = block_offsets[i]
142 cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
143 cloned_value_cache[block_idx, :, :, block_offset] = value[i]
144
145 assert torch.allclose(key_cache, cloned_key_cache)
146 assert torch.allclose(value_cache, cloned_value_cache)
115 -
116 -
117 -@torch.inference_mode()
118 -def run_gather_cached_kv(
119 - num_tokens: int,
120 - num_heads: int,
121 - head_size: int,
122 - block_size: int,
123 - num_blocks: int,
124 - dtype: torch.dtype,
125 -) -> None:
126 - num_slots = block_size * num_blocks
127 - slot_mapping = random.sample(range(num_slots), num_tokens)
128 - slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
129 -
130 - qkv = torch.randn(num_tokens,
131 - 3,
132 - num_heads,
133 - head_size,
134 - dtype=dtype,
135 - device='cuda')
136 - _, key, value = qkv.unbind(dim=1)
137 -
138 - qkv_clone = qkv.clone()
139 - _, cloned_key, cloned_value = qkv_clone.unbind(dim=1)
140 -
141 - x = 16 // torch.tensor([], dtype=dtype).element_size()
142 - key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
143 - key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
144 -
145 - value_cache_shape = (num_blocks, num_heads, head_size, block_size)
146 - value_cache = torch.randn(size=value_cache_shape,
147 - dtype=dtype,
148 - device='cuda')
149 -
150 - cache_ops.gather_cached_kv(key, value, key_cache, value_cache,
151 - slot_mapping)
152 -
153 - # Reference implementation.
154 - for i in range(num_tokens):
155 - reshaped_key = cloned_key.reshape(num_tokens, num_heads,
156 - head_size // x, x)
157 - block_idx = torch.div(slot_mapping[i],
158 - block_size,
159 - rounding_mode='floor')
160 - block_offset = slot_mapping[i] % block_size
161 - reshaped_key[i] = key_cache[block_idx, :, :, block_offset, :]
162 - cloned_value[i] = value_cache[block_idx, :, :, block_offset]
163 -
164 - assert torch.allclose(key, cloned_key)
165 - assert torch.allclose(value, cloned_value)
166 -
167 -
168 -def test_copy_blocks() -> None:
169 - for dtype in [torch.half, torch.bfloat16, torch.float]:
170 - run_copy_blocks(num_mappings=23,
171 - num_layers=7,
172 - num_heads=17,
173 - head_size=16,
174 - block_size=8,
175 - num_blocks=1024,
176 - dtype=dtype)
177 -
178 -
179 -def test_reshape_and_cache() -> None:
180 - for dtype in [torch.half, torch.bfloat16, torch.float]:
181 - run_reshape_and_cache(num_tokens=3,
182 - num_heads=2,
183 - head_size=16,
184 - block_size=8,
185 - num_blocks=2,
186 - dtype=dtype)
187 -
188 -
189 -def test_gather_cached_kv() -> None:
190 - for dtype in [torch.half, torch.bfloat16, torch.float]:
191 - run_gather_cached_kv(num_tokens=3,
192 - num_heads=2,
193 - head_size=16,
194 - block_size=8,
195 - num_blocks=2,
196 - dtype=dtype)
tests/kernels/test_layernorm.py
+24 -22
@@ -1,35 +1,50 @@
1 +import pytest
2 import torch
3 import torch.nn as nn
4
5 from vllm import layernorm_ops
6
7 +DTYPES = [torch.half, torch.bfloat16, torch.float]
8 +HIDDEN_SIZES = [67, 768, 2048, 5120, 8192] # Arbitrary values for testing
9 +NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
10 +SEEDS = [0]
11 +
12
13 class RefRMSNorm(nn.Module):
14
15 def __init__(self, hidden_size, eps=1e-6):
16 super().__init__()
17 weight = torch.empty(hidden_size)
12 - weight.uniform_(-1e-3, 1e-3)
18 + weight.normal_(mean=1.0, std=0.1)
19 self.weight = nn.Parameter(weight)
20 self.variance_epsilon = eps
21
22 def forward(self, hidden_states):
17 - variance = hidden_states.to(torch.float32).pow(2).mean(-1,
18 - keepdim=True)
23 + input_dtype = hidden_states.dtype
24 + hidden_states = hidden_states.to(torch.float32)
25 + variance = hidden_states.pow(2).mean(-1, keepdim=True)
26 hidden_states = hidden_states * torch.rsqrt(variance +
27 self.variance_epsilon)
21 - if self.weight.dtype in [torch.half, torch.float16, torch.bfloat16]:
22 - hidden_states = hidden_states.to(self.weight.dtype)
23 - return self.weight * hidden_states
28 + return self.weight * hidden_states.to(input_dtype)
29
30
31 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
32 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
33 +@pytest.mark.parametrize("dtype", DTYPES)
34 +@pytest.mark.parametrize("seed", SEEDS)
35 @torch.inference_mode()
27 -def run_rms_norm(
36 +def test_rms_norm(
37 num_tokens: int,
38 hidden_size: int,
39 dtype: torch.dtype,
40 + seed: int,
41 ) -> None:
32 - x = torch.randn(num_tokens, hidden_size, dtype=dtype, device='cuda')
42 + torch.random.manual_seed(seed)
43 + torch.cuda.manual_seed(seed)
44 +
45 + scale = float(hidden_size**-0.5)
46 + x = torch.empty(num_tokens, hidden_size, dtype=dtype, device="cuda")
47 + x.uniform_(-scale, scale)
48 ref = RefRMSNorm(hidden_size).to(dtype).cuda()
49
50 out = torch.empty_like(x)
@@ -40,17 +55,4 @@ def run_rms_norm(
55 ref.variance_epsilon,
56 )
57 ref_out = ref(x)
43 - assert torch.allclose(out, ref_out, atol=1e-3, rtol=1e-5)
44 -
45 -
46 -def test_rms_norm() -> None:
47 - for dtype in [torch.half, torch.bfloat16, torch.float]:
48 - for num_tokens in [7, 128, 2048]:
49 - for hidden_size in [13, 64, 1024, 5120]:
50 - print(f'Testing RMS kernel with dtype={dtype}, num_tokens='
51 - f'{num_tokens}, hidden_size={hidden_size}')
52 - run_rms_norm(
53 - num_tokens=num_tokens,
54 - hidden_size=hidden_size,
55 - dtype=dtype,
56 - )
58 + assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-5)
tests/kernels/test_pos_encoding.py
+27 -21
@@ -1,11 +1,19 @@
1 -from typing import Tuple
1 +from typing import Optional, Tuple
2
3 +import pytest
4 import torch
5 import torch.nn as nn
6 import torch.nn.functional as F
7
8 from vllm import pos_encoding_ops
9
10 +DTYPES = [torch.half, torch.bfloat16, torch.float]
11 +HEAD_SIZES = [64, 80, 96, 112, 128, 256]
12 +ROTARY_DIMS = [None, 32] # None means rotary dim == head size
13 +NUM_HEADS = [7, 12, 40, 52] # Arbitrary values for testing
14 +NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
15 +SEEDS = [0]
16 +
17
18 def rotate_half(x: torch.Tensor) -> torch.Tensor:
19 x1 = x[..., :x.shape[-1] // 2]
@@ -74,16 +82,28 @@ class RefRotaryEmbeddingNeox(nn.Module):
82 return query, key
83
84
85 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
86 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
87 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
88 +@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
89 +@pytest.mark.parametrize("dtype", DTYPES)
90 +@pytest.mark.parametrize("seed", SEEDS)
91 @torch.inference_mode()
78 -def run_rotary_embedding_neox(
92 +def test_rotary_embedding_neox(
93 num_tokens: int,
94 num_heads: int,
95 head_size: int,
82 - max_position: int,
83 - rotary_dim: int,
96 + rotary_dim: Optional[int],
97 dtype: torch.dtype,
98 + seed: int,
99 + max_position: int = 8192,
100 base: int = 10000,
101 ) -> None:
102 + if rotary_dim is None:
103 + rotary_dim = head_size
104 + torch.random.manual_seed(seed)
105 + torch.cuda.manual_seed(seed)
106 +
107 positions = torch.randint(0, max_position, (num_tokens, ), device='cuda')
108 query = torch.randn(num_tokens,
109 num_heads * head_size,
@@ -97,7 +117,7 @@ def run_rotary_embedding_neox(
117 # Create the rotary embedding.
118 inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
119 t = torch.arange(max_position).float()
100 - freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
120 + freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
121 cos = freqs.cos()
122 sin = freqs.sin()
123 cos_sin_cache = torch.cat((cos, sin), dim=-1)
@@ -129,19 +149,5 @@ def run_rotary_embedding_neox(
149 ref_key = ref_key.view(num_tokens, num_heads * head_size)
150
151 # Compare the results.
132 - assert torch.allclose(out_query, ref_query, atol=1e-3, rtol=1e-5)
133 - assert torch.allclose(out_key, ref_key, atol=1e-3, rtol=1e-5)
134 -
135 -
136 -def test_rotary_embedding_neox() -> None:
137 - for dtype in [torch.half, torch.bfloat16, torch.float]:
138 - for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
139 - print(f'Running tests for head_size={head_size} and dtype={dtype}')
140 - run_rotary_embedding_neox(
141 - num_tokens=2145,
142 - num_heads=5,
143 - head_size=head_size,
144 - max_position=8192,
145 - rotary_dim=head_size,
146 - dtype=dtype,
147 - )
152 + assert torch.allclose(out_query, ref_query, atol=1e-5, rtol=1e-5)
153 + assert torch.allclose(out_key, ref_key, atol=1e-5, rtol=1e-5)