A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Use pytest format for unit tests (#107)
Woosuk Kwon committed
May 17, 2023 at 17:11 UTC
825d8892b54af80516ce98a89b595018f756a6d3
5 files changed
+43
-37
tests/kernels/test_activation.py
renamed
+4
-4
@@ -10,7 +10,7 @@ def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
10
11
12
@torch.inference_mode()
13
-def test_silu_and_mul(
13
+def run_silu_and_mul(
14
num_tokens: int,
15
d: int,
16
dtype: torch.dtype,
@@ -22,9 +22,9 @@ def test_silu_and_mul(
22
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
23
24
25
-if __name__ == '__main__':
25
+def test_silu_and_mul() -> None:
26
for dtype in [torch.half, torch.bfloat16, torch.float]:
27
for num_tokens in [7, 83, 2048]:
28
- for d in [512, 4096, 13824]:
28
+ for d in [512, 4096, 5120, 13824]:
29
print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
30
- test_silu_and_mul(num_tokens, d, dtype)
30
+ run_silu_and_mul(num_tokens, d, dtype)
tests/kernels/test_attention.py
renamed
+14
-14
@@ -8,6 +8,7 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
from cacheflow import attention_ops
9
10
MAX_SEQ_LEN = 4096
11
+TEST_SEED = 0
12
13
14
def ref_masked_attention(
@@ -155,7 +156,8 @@ def ref_multi_query_cached_kv_attention(
156
return ref_output
157
158
158
-def test_single_query_cached_kv_attention(
159
+@torch.inference_mode()
160
+def run_single_query_cached_kv_attention(
161
num_tokens: int,
162
num_heads: int,
163
head_size: int,
@@ -223,7 +225,8 @@ def test_single_query_cached_kv_attention(
225
assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
226
227
226
-def test_multi_query_kv_attention(
228
+@torch.inference_mode()
229
+def run_multi_query_kv_attention(
230
num_seqs: int,
231
num_heads: int,
232
head_size: int,
@@ -264,19 +267,16 @@ def test_multi_query_kv_attention(
267
assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
268
269
267
-@torch.inference_mode()
268
-def test_attention(seed: int) -> None:
269
- # NOTE(woosuk): Even when the seed is fixed, there is a chance that
270
- # the test fails due to the precision issue. Re-run the test if it fails.
271
- torch.random.manual_seed(seed)
272
- torch.cuda.manual_seed(seed)
270
+def test_single_query_cached_kv_attention() -> None:
271
+ torch.random.manual_seed(TEST_SEED)
272
+ torch.cuda.manual_seed(TEST_SEED)
273
for dtype in [torch.half, torch.bfloat16]:
274
for block_size in [8, 16, 32, 64]:
275
for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
276
print(f'Testing single_query_cached_kv_attention with '
277
f'dtype={dtype}, block_size={block_size}, '
278
f'head_size={head_size}')
279
- test_single_query_cached_kv_attention(
279
+ run_single_query_cached_kv_attention(
280
num_tokens=37,
281
num_heads=3,
282
head_size=head_size,
@@ -285,17 +285,17 @@ def test_attention(seed: int) -> None:
285
dtype=dtype,
286
)
287
288
+
289
+def test_multi_query_kv_attention() -> None:
290
+ torch.random.manual_seed(TEST_SEED)
291
+ torch.cuda.manual_seed(TEST_SEED)
292
for dtype in [torch.half, torch.bfloat16]:
293
for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
294
print(f'Testing multi_query_kv_attention with dtype={dtype}, '
295
f'head_size={head_size}')
292
- test_multi_query_kv_attention(
296
+ run_multi_query_kv_attention(
297
num_seqs=5,
298
num_heads=3,
299
head_size=head_size,
300
dtype=dtype,
301
)
298
-
299
-
300
-if __name__ == '__main__':
301
- test_attention(seed=0)
tests/kernels/test_cache.py
renamed
+18
-12
@@ -5,7 +5,8 @@ import torch
5
from cacheflow import cache_ops
6
7
8
-def test_copy_blocks(
8
+@torch.inference_mode()
9
+def run_copy_blocks(
10
num_mappings: int,
11
num_layers: int,
12
num_heads: int,
@@ -60,7 +61,8 @@ def test_copy_blocks(
61
assert torch.allclose(value_cache, cloned_value_cache)
62
63
63
-def test_reshape_and_cache(
64
+@torch.inference_mode()
65
+def run_reshape_and_cache(
66
num_tokens: int,
67
num_heads: int,
68
head_size: int,
@@ -99,7 +101,8 @@ def test_reshape_and_cache(
101
assert torch.allclose(value_cache, cloned_value_cache)
102
103
102
-def test_gather_cached_kv(
104
+@torch.inference_mode()
105
+def run_gather_cached_kv(
106
num_tokens: int,
107
num_heads: int,
108
head_size: int,
@@ -140,19 +143,22 @@ def test_gather_cached_kv(
143
assert torch.allclose(value, cloned_value)
144
145
143
-@torch.inference_mode()
144
-def test_cache() -> None:
146
+def test_copy_blocks() -> None:
147
for dtype in [torch.half, torch.bfloat16, torch.float]:
146
- test_copy_blocks(
148
+ run_copy_blocks(
149
num_mappings=23, num_layers=7, num_heads=17, head_size=16,
150
block_size=8, num_blocks=1024, dtype=dtype)
149
- test_reshape_and_cache(
150
- num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
151
- dtype=dtype)
152
- test_gather_cached_kv(
151
+
152
+
153
+def test_reshape_and_cache() -> None:
154
+ for dtype in [torch.half, torch.bfloat16, torch.float]:
155
+ run_reshape_and_cache(
156
num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
157
dtype=dtype)
158
159
157
-if __name__ == '__main__':
158
- test_cache()
160
+def test_gather_cached_kv() -> None:
161
+ for dtype in [torch.half, torch.bfloat16, torch.float]:
162
+ run_gather_cached_kv(
163
+ num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
164
+ dtype=dtype)
tests/kernels/test_layernorm.py
renamed
+3
-3
@@ -22,7 +22,7 @@ class RefRMSNorm(nn.Module):
22
23
24
@torch.inference_mode()
25
-def test_rms_norm(
25
+def run_rms_norm(
26
num_tokens: int,
27
hidden_size: int,
28
dtype: torch.dtype,
@@ -41,13 +41,13 @@ def test_rms_norm(
41
assert torch.allclose(out, ref_out, atol=1e-3, rtol=1e-5)
42
43
44
-if __name__ == '__main__':
44
+def test_rms_norm() -> None:
45
for dtype in [torch.half, torch.bfloat16, torch.float]:
46
for num_tokens in [7, 128, 2048]:
47
for hidden_size in [13, 64, 1024, 5120]:
48
print(f'Testing RMS kernel with dtype={dtype}, num_tokens='
49
f'{num_tokens}, hidden_size={hidden_size}')
50
- test_rms_norm(
50
+ run_rms_norm(
51
num_tokens=num_tokens,
52
hidden_size=hidden_size,
53
dtype=dtype,
tests/kernels/test_pos_encoding.py
renamed
+4
-4
@@ -76,7 +76,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
76
77
78
@torch.inference_mode()
79
-def test_rotary_embedding_neox(
79
+def run_rotary_embedding_neox(
80
num_tokens: int,
81
num_heads: int,
82
head_size: int,
@@ -128,15 +128,15 @@ def test_rotary_embedding_neox(
128
assert torch.allclose(out_key, ref_key, atol=1e-3, rtol=1e-5)
129
130
131
-if __name__ == '__main__':
131
+def test_rotary_embedding_neox() -> None:
132
for dtype in [torch.half, torch.bfloat16, torch.float]:
133
for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
134
print(f'Running tests for head_size={head_size} and dtype={dtype}')
135
- test_rotary_embedding_neox(
135
+ run_rotary_embedding_neox(
136
num_tokens=2145,
137
num_heads=5,
138
head_size=head_size,
139
max_position=8192,
140
- rotary_dim=int(head_size * 0.25),
140
+ rotary_dim=head_size,
141
dtype=dtype,
142
)