@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 825d8892b5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Use pytest format for unit tests (#107)

Woosuk Kwon committed May 17, 2023 at 17:11 UTC 825d8892b54af80516ce98a89b595018f756a6d3
5 files changed +43 -37
tests/kernels/test_activation.py renamed
+4 -4
@@ -10,7 +10,7 @@ def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
10
11
12 @torch.inference_mode()
13 -def test_silu_and_mul(
13 +def run_silu_and_mul(
14 num_tokens: int,
15 d: int,
16 dtype: torch.dtype,
@@ -22,9 +22,9 @@ def test_silu_and_mul(
22 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
23
24
25 -if __name__ == '__main__':
25 +def test_silu_and_mul() -> None:
26 for dtype in [torch.half, torch.bfloat16, torch.float]:
27 for num_tokens in [7, 83, 2048]:
28 - for d in [512, 4096, 13824]:
28 + for d in [512, 4096, 5120, 13824]:
29 print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
30 - test_silu_and_mul(num_tokens, d, dtype)
30 + run_silu_and_mul(num_tokens, d, dtype)
tests/kernels/test_attention.py renamed
+14 -14
@@ -8,6 +8,7 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8 from cacheflow import attention_ops
9
10 MAX_SEQ_LEN = 4096
11 +TEST_SEED = 0
12
13
14 def ref_masked_attention(
@@ -155,7 +156,8 @@ def ref_multi_query_cached_kv_attention(
156 return ref_output
157
158
158 -def test_single_query_cached_kv_attention(
159 +@torch.inference_mode()
160 +def run_single_query_cached_kv_attention(
161 num_tokens: int,
162 num_heads: int,
163 head_size: int,
@@ -223,7 +225,8 @@ def test_single_query_cached_kv_attention(
225 assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
226
227
226 -def test_multi_query_kv_attention(
228 +@torch.inference_mode()
229 +def run_multi_query_kv_attention(
230 num_seqs: int,
231 num_heads: int,
232 head_size: int,
@@ -264,19 +267,16 @@ def test_multi_query_kv_attention(
267 assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
268
269
267 -@torch.inference_mode()
268 -def test_attention(seed: int) -> None:
269 - # NOTE(woosuk): Even when the seed is fixed, there is a chance that
270 - # the test fails due to the precision issue. Re-run the test if it fails.
271 - torch.random.manual_seed(seed)
272 - torch.cuda.manual_seed(seed)
270 +def test_single_query_cached_kv_attention() -> None:
271 + torch.random.manual_seed(TEST_SEED)
272 + torch.cuda.manual_seed(TEST_SEED)
273 for dtype in [torch.half, torch.bfloat16]:
274 for block_size in [8, 16, 32, 64]:
275 for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
276 print(f'Testing single_query_cached_kv_attention with '
277 f'dtype={dtype}, block_size={block_size}, '
278 f'head_size={head_size}')
279 - test_single_query_cached_kv_attention(
279 + run_single_query_cached_kv_attention(
280 num_tokens=37,
281 num_heads=3,
282 head_size=head_size,
@@ -285,17 +285,17 @@ def test_attention(seed: int) -> None:
285 dtype=dtype,
286 )
287
288 +
289 +def test_multi_query_kv_attention() -> None:
290 + torch.random.manual_seed(TEST_SEED)
291 + torch.cuda.manual_seed(TEST_SEED)
292 for dtype in [torch.half, torch.bfloat16]:
293 for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
294 print(f'Testing multi_query_kv_attention with dtype={dtype}, '
295 f'head_size={head_size}')
292 - test_multi_query_kv_attention(
296 + run_multi_query_kv_attention(
297 num_seqs=5,
298 num_heads=3,
299 head_size=head_size,
300 dtype=dtype,
301 )
298 -
299 -
300 -if __name__ == '__main__':
301 - test_attention(seed=0)
tests/kernels/test_cache.py renamed
+18 -12
@@ -5,7 +5,8 @@ import torch
5 from cacheflow import cache_ops
6
7
8 -def test_copy_blocks(
8 +@torch.inference_mode()
9 +def run_copy_blocks(
10 num_mappings: int,
11 num_layers: int,
12 num_heads: int,
@@ -60,7 +61,8 @@ def test_copy_blocks(
61 assert torch.allclose(value_cache, cloned_value_cache)
62
63
63 -def test_reshape_and_cache(
64 +@torch.inference_mode()
65 +def run_reshape_and_cache(
66 num_tokens: int,
67 num_heads: int,
68 head_size: int,
@@ -99,7 +101,8 @@ def test_reshape_and_cache(
101 assert torch.allclose(value_cache, cloned_value_cache)
102
103
102 -def test_gather_cached_kv(
104 +@torch.inference_mode()
105 +def run_gather_cached_kv(
106 num_tokens: int,
107 num_heads: int,
108 head_size: int,
@@ -140,19 +143,22 @@ def test_gather_cached_kv(
143 assert torch.allclose(value, cloned_value)
144
145
143 -@torch.inference_mode()
144 -def test_cache() -> None:
146 +def test_copy_blocks() -> None:
147 for dtype in [torch.half, torch.bfloat16, torch.float]:
146 - test_copy_blocks(
148 + run_copy_blocks(
149 num_mappings=23, num_layers=7, num_heads=17, head_size=16,
150 block_size=8, num_blocks=1024, dtype=dtype)
149 - test_reshape_and_cache(
150 - num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
151 - dtype=dtype)
152 - test_gather_cached_kv(
151 +
152 +
153 +def test_reshape_and_cache() -> None:
154 + for dtype in [torch.half, torch.bfloat16, torch.float]:
155 + run_reshape_and_cache(
156 num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
157 dtype=dtype)
158
159
157 -if __name__ == '__main__':
158 - test_cache()
160 +def test_gather_cached_kv() -> None:
161 + for dtype in [torch.half, torch.bfloat16, torch.float]:
162 + run_gather_cached_kv(
163 + num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
164 + dtype=dtype)
tests/kernels/test_layernorm.py renamed
+3 -3
@@ -22,7 +22,7 @@ class RefRMSNorm(nn.Module):
22
23
24 @torch.inference_mode()
25 -def test_rms_norm(
25 +def run_rms_norm(
26 num_tokens: int,
27 hidden_size: int,
28 dtype: torch.dtype,
@@ -41,13 +41,13 @@ def test_rms_norm(
41 assert torch.allclose(out, ref_out, atol=1e-3, rtol=1e-5)
42
43
44 -if __name__ == '__main__':
44 +def test_rms_norm() -> None:
45 for dtype in [torch.half, torch.bfloat16, torch.float]:
46 for num_tokens in [7, 128, 2048]:
47 for hidden_size in [13, 64, 1024, 5120]:
48 print(f'Testing RMS kernel with dtype={dtype}, num_tokens='
49 f'{num_tokens}, hidden_size={hidden_size}')
50 - test_rms_norm(
50 + run_rms_norm(
51 num_tokens=num_tokens,
52 hidden_size=hidden_size,
53 dtype=dtype,
tests/kernels/test_pos_encoding.py renamed
+4 -4
@@ -76,7 +76,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
76
77
78 @torch.inference_mode()
79 -def test_rotary_embedding_neox(
79 +def run_rotary_embedding_neox(
80 num_tokens: int,
81 num_heads: int,
82 head_size: int,
@@ -128,15 +128,15 @@ def test_rotary_embedding_neox(
128 assert torch.allclose(out_key, ref_key, atol=1e-3, rtol=1e-5)
129
130
131 -if __name__ == '__main__':
131 +def test_rotary_embedding_neox() -> None:
132 for dtype in [torch.half, torch.bfloat16, torch.float]:
133 for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
134 print(f'Running tests for head_size={head_size} and dtype={dtype}')
135 - test_rotary_embedding_neox(
135 + run_rotary_embedding_neox(
136 num_tokens=2145,
137 num_heads=5,
138 head_size=head_size,
139 max_position=8192,
140 - rotary_dim=int(head_size * 0.25),
140 + rotary_dim=head_size,
141 dtype=dtype,
142 )