@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 235366fe2e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[CI] Prune back the number of tests in tests/kernels/* (#9932)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 5, 2024 at 16:02 UTC 235366fe2eb3144321978e181af94487f0215595
10 files changed +60 -36
tests/kernels/test_activation.py
+1 -1
@@ -14,7 +14,7 @@ from .allclose_default import get_default_atol, get_default_rtol
14
15 DTYPES = [torch.half, torch.bfloat16, torch.float]
16 NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
17 -D = [512, 4096, 5120, 13824] # Arbitrary values for testing
17 +D = [512, 13824] # Arbitrary values for testing
18 SEEDS = [0]
19 CUDA_DEVICES = [
20 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
tests/kernels/test_attention.py
+1 -1
@@ -33,7 +33,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
33
34 # FlashAttention forward only supports head dimension at most 128
35 # https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
36 -HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
36 +HEAD_SIZES = [64, 80, 120, 256]
37
38 BLOCK_SIZES = [16, 32]
39 USE_ALIBI = [False, True]
tests/kernels/test_awq_marlin.py
+10 -6
@@ -14,13 +14,17 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
14 awq_marlin_quantize)
15 from vllm.scalar_type import scalar_types
16
17 +NUM_EXPERTS = [8, 64]
18 +TOP_KS = [2, 6]
19 +GROUP_SIZES = [-1, 32, 128]
20
18 -@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
19 -@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
20 -@pytest.mark.parametrize("k", [128, 1024, 512])
21 -@pytest.mark.parametrize("e", [8, 64])
22 -@pytest.mark.parametrize("topk", [2, 6])
23 -@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
21 +
22 +@pytest.mark.parametrize("m", [1, 33, 64, 222])
23 +@pytest.mark.parametrize("n", [128, 2048])
24 +@pytest.mark.parametrize("k", [128, 1024])
25 +@pytest.mark.parametrize("e", NUM_EXPERTS)
26 +@pytest.mark.parametrize("topk", TOP_KS)
27 +@pytest.mark.parametrize("group_size", GROUP_SIZES)
28 @pytest.mark.skipif(not (ops.supports_moe_ops
29 and hasattr(torch.ops._moe_C, "marlin_gemm_moe")),
30 reason="Marlin is not supported on this GPU type.")
tests/kernels/test_blocksparse_attention.py
+3 -3
@@ -25,10 +25,10 @@ PARTITION_SIZE = 512
25 DTYPES = [torch.half, torch.bfloat16]
26 NUM_GEN_SEQS = [3] # Arbitrary values for testing
27 NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
28 -NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
28 +NUM_HEADS = [(40, 40)] # Arbitrary values for testing
29
30 HEAD_SIZES = [64, 112]
31 -BLOCK_SIZES = [16, 32]
31 +BLOCK_SIZES = [16]
32 USE_ALIBI = [False, True]
33 KV_CACHE_DTYPE = ["auto", "fp8"]
34 SEEDS = [0]
@@ -37,7 +37,7 @@ BLOCKSPARSE_LOCAL_BLOCKS = [16]
37 BLOCKSPARSE_VERT_STRIDES = [8]
38
39 BLOCKSPARSE_BLOCK_SIZES = [64]
40 -BLOCKSPARSE_HEADS_SLIDINGS = [0, 2, -1]
40 +BLOCKSPARSE_HEADS_SLIDINGS = [2, -1]
41 BLOCKSPARSE_HOMO_HEADS = [True, False]
42
43
tests/kernels/test_cache.py
+1 -1
@@ -13,7 +13,7 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
13 NUM_TOKENS = [42] # Arbitrary values for testing
14 NUM_LAYERS = [1] # Arbitrary values for testing
15 NUM_HEADS = [8] # Arbitrary values for testing
16 -HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
16 +HEAD_SIZES = [64, 80, 120, 256]
17 BLOCK_SIZES = [8, 16, 32]
18
19 # Arbitrary values for testing
tests/kernels/test_cutlass.py
+24 -6
@@ -11,6 +11,28 @@ from tests.kernels.utils import opcheck
11 from vllm import _custom_ops as ops
12 from vllm.platforms import current_platform
13
14 +MNK_FACTORS = [
15 + (1, 256, 128),
16 + (1, 16384, 1024),
17 + (1, 24576, 496),
18 + (16, 256, 496),
19 + (16, 16384, 128),
20 + (16, 24576, 4096),
21 + (32, 8192, 4096),
22 + (32, 16384, 4096),
23 + (33, 1024, 1024),
24 + (33, 8192, 128),
25 + (64, 2048, 496),
26 + (64, 16384, 1024),
27 + (100, 8192, 496),
28 + (128, 32768, 4096),
29 + (256, 4096, 4096),
30 + (512, 256, 1024),
31 + (512, 8192, 4096),
32 + (512, 16384, 128),
33 + (512, 24576, 128),
34 +]
35 +
36 CUDA_DEVICES = [
37 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
38 ]
@@ -116,9 +138,7 @@ def cutlass_int8_gemm_helper(m: int,
138 (out, a, b, scale_a, scale_b, bias))
139
140
119 -@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 100, 33])
120 -@pytest.mark.parametrize("n", [2048, 4096, 8192, 16384, 24576, 256, 1024])
121 -@pytest.mark.parametrize("k", [128, 496, 1024])
141 +@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
142 @pytest.mark.parametrize("per_act_token", [True, False])
143 @pytest.mark.parametrize("per_out_ch", [True, False])
144 @pytest.mark.parametrize("use_bias", [True, False])
@@ -129,9 +149,7 @@ def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
149 cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
150
151
132 -@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 33, 1])
133 -@pytest.mark.parametrize("n", [2048, 8192, 16384, 256, 1024])
134 -@pytest.mark.parametrize("k", [128, 496, 1024])
152 +@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
153 @pytest.mark.parametrize("per_act_token", [True, False])
154 @pytest.mark.parametrize("per_out_ch", [True, False])
155 @pytest.mark.parametrize("use_bias", [True, False])
tests/kernels/test_int8_quant.py
+3 -4
@@ -7,11 +7,10 @@ from vllm._custom_ops import scaled_int8_quant
7 from vllm.platforms import current_platform
8
9 DTYPES = [torch.half, torch.bfloat16, torch.float]
10 -HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
11 - 8193] # Arbitrary values for testing
10 +HIDDEN_SIZES = [16, 67, 768, 5137, 8193] # Arbitrary values for testing
11 NUM_TOKENS = [1, 7, 83, 4096] # Arbitrary values for testing
12 SEEDS = [0]
14 -SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
13 +SCALE = [0.1, 2.1]
14
15
16 def opcheck_int8_quant_static(output, input, scale, azp=None):
@@ -132,7 +131,7 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
131 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
132 @pytest.mark.parametrize("dtype", DTYPES)
133 @pytest.mark.parametrize("seed", SEEDS)
135 -@pytest.mark.parametrize("scale", SCALE[2:]) # Reduce test time
134 +@pytest.mark.parametrize("scale", SCALE)
135 @pytest.mark.parametrize("azp", [-255, 54])
136 @torch.inference_mode()
137 def test_static_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
tests/kernels/test_marlin_gemm.py
+1 -1
@@ -35,7 +35,7 @@ K_FULL_OPTS = [False, True]
35 USE_FP32_REDUCE_OPTS = [False, True]
36
37 MARLIN_K_CHUNKS = [128]
38 -MARLIN_N_CHUNKS = [64, 128, 256]
38 +MARLIN_N_CHUNKS = [64, 256]
39
40 MARLIN_24_K_CHUNKS = [128]
41 MARLIN_24_N_CHUNKS = [512]
tests/kernels/test_moe.py
+13 -10
@@ -20,12 +20,15 @@ from vllm.model_executor.models.mixtral import MixtralMoE
20 from vllm.platforms import current_platform
21 from vllm.scalar_type import scalar_types
22
23 +NUM_EXPERTS = [8, 64]
24 +TOP_KS = [2, 6]
25
24 -@pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
25 -@pytest.mark.parametrize("n", [2048, 256, 1024])
26 +
27 +@pytest.mark.parametrize("m", [1, 33, 64, 222, 1024 * 128])
28 +@pytest.mark.parametrize("n", [128, 1024, 2048])
29 @pytest.mark.parametrize("k", [128, 511, 1024])
27 -@pytest.mark.parametrize("e", [8, 64])
28 -@pytest.mark.parametrize("topk", [2, 6])
30 +@pytest.mark.parametrize("e", NUM_EXPERTS)
31 +@pytest.mark.parametrize("topk", TOP_KS)
32 @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
33 def test_fused_moe(
34 m: int,
@@ -93,12 +96,12 @@ def test_mixtral_moe(dtype: torch.dtype):
96 atol=mixtral_moe_tol[dtype])
97
98
96 -@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
97 -@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
98 -@pytest.mark.parametrize("k", [128, 1024, 512])
99 -@pytest.mark.parametrize("e", [8, 64])
100 -@pytest.mark.parametrize("topk", [2, 6])
101 -@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
99 +@pytest.mark.parametrize("m", [1, 33, 64, 222])
100 +@pytest.mark.parametrize("n", [128, 2048])
101 +@pytest.mark.parametrize("k", [128, 1024])
102 +@pytest.mark.parametrize("e", NUM_EXPERTS)
103 +@pytest.mark.parametrize("topk", TOP_KS)
104 +@pytest.mark.parametrize("group_size", [-1, 32, 128])
105 @pytest.mark.parametrize("act_order", [True, False])
106 @pytest.mark.parametrize("num_bits", [4, 8])
107 @pytest.mark.parametrize("is_k_full", [True, False])
tests/kernels/test_pos_encoding.py
+3 -3
@@ -11,10 +11,10 @@ from .allclose_default import get_default_atol, get_default_rtol
11
12 IS_NEOX_STYLE = [True, False]
13 DTYPES = [torch.half, torch.bfloat16, torch.float]
14 -HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
14 +HEAD_SIZES = [64, 80, 112, 120, 256]
15 ROTARY_DIMS = [None, 32] # None means rotary dim == head size
16 -NUM_HEADS = [7, 17] # Arbitrary values for testing
17 -BATCH_SIZES = [1, 5] # Arbitrary values for testing
16 +NUM_HEADS = [17] # Arbitrary values for testing
17 +BATCH_SIZES = [5] # Arbitrary values for testing
18 SEQ_LENS = [11, 8192] # Arbitrary values for testing
19 SEEDS = [0]
20 CUDA_DEVICES = [