A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI] Prune back the number of tests in tests/kernels/* (#9932)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 5, 2024 at 16:02 UTC
235366fe2eb3144321978e181af94487f0215595
10 files changed
+60
-36
tests/kernels/test_activation.py
+1
-1
@@ -14,7 +14,7 @@ from .allclose_default import get_default_atol, get_default_rtol
14
15
DTYPES = [torch.half, torch.bfloat16, torch.float]
16
NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
17
-D = [512, 4096, 5120, 13824] # Arbitrary values for testing
17
+D = [512, 13824] # Arbitrary values for testing
18
SEEDS = [0]
19
CUDA_DEVICES = [
20
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
tests/kernels/test_attention.py
+1
-1
@@ -33,7 +33,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
33
34
# FlashAttention forward only supports head dimension at most 128
35
# https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
36
-HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
36
+HEAD_SIZES = [64, 80, 120, 256]
37
38
BLOCK_SIZES = [16, 32]
39
USE_ALIBI = [False, True]
tests/kernels/test_awq_marlin.py
+10
-6
@@ -14,13 +14,17 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
14
awq_marlin_quantize)
15
from vllm.scalar_type import scalar_types
16
17
+NUM_EXPERTS = [8, 64]
18
+TOP_KS = [2, 6]
19
+GROUP_SIZES = [-1, 32, 128]
20
18
-@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
19
-@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
20
-@pytest.mark.parametrize("k", [128, 1024, 512])
21
-@pytest.mark.parametrize("e", [8, 64])
22
-@pytest.mark.parametrize("topk", [2, 6])
23
-@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
21
+
22
+@pytest.mark.parametrize("m", [1, 33, 64, 222])
23
+@pytest.mark.parametrize("n", [128, 2048])
24
+@pytest.mark.parametrize("k", [128, 1024])
25
+@pytest.mark.parametrize("e", NUM_EXPERTS)
26
+@pytest.mark.parametrize("topk", TOP_KS)
27
+@pytest.mark.parametrize("group_size", GROUP_SIZES)
28
@pytest.mark.skipif(not (ops.supports_moe_ops
29
and hasattr(torch.ops._moe_C, "marlin_gemm_moe")),
30
reason="Marlin is not supported on this GPU type.")
tests/kernels/test_blocksparse_attention.py
+3
-3
@@ -25,10 +25,10 @@ PARTITION_SIZE = 512
25
DTYPES = [torch.half, torch.bfloat16]
26
NUM_GEN_SEQS = [3] # Arbitrary values for testing
27
NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
28
-NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
28
+NUM_HEADS = [(40, 40)] # Arbitrary values for testing
29
30
HEAD_SIZES = [64, 112]
31
-BLOCK_SIZES = [16, 32]
31
+BLOCK_SIZES = [16]
32
USE_ALIBI = [False, True]
33
KV_CACHE_DTYPE = ["auto", "fp8"]
34
SEEDS = [0]
@@ -37,7 +37,7 @@ BLOCKSPARSE_LOCAL_BLOCKS = [16]
37
BLOCKSPARSE_VERT_STRIDES = [8]
38
39
BLOCKSPARSE_BLOCK_SIZES = [64]
40
-BLOCKSPARSE_HEADS_SLIDINGS = [0, 2, -1]
40
+BLOCKSPARSE_HEADS_SLIDINGS = [2, -1]
41
BLOCKSPARSE_HOMO_HEADS = [True, False]
42
43
tests/kernels/test_cache.py
+1
-1
@@ -13,7 +13,7 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
13
NUM_TOKENS = [42] # Arbitrary values for testing
14
NUM_LAYERS = [1] # Arbitrary values for testing
15
NUM_HEADS = [8] # Arbitrary values for testing
16
-HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
16
+HEAD_SIZES = [64, 80, 120, 256]
17
BLOCK_SIZES = [8, 16, 32]
18
19
# Arbitrary values for testing
tests/kernels/test_cutlass.py
+24
-6
@@ -11,6 +11,28 @@ from tests.kernels.utils import opcheck
11
from vllm import _custom_ops as ops
12
from vllm.platforms import current_platform
13
14
+MNK_FACTORS = [
15
+ (1, 256, 128),
16
+ (1, 16384, 1024),
17
+ (1, 24576, 496),
18
+ (16, 256, 496),
19
+ (16, 16384, 128),
20
+ (16, 24576, 4096),
21
+ (32, 8192, 4096),
22
+ (32, 16384, 4096),
23
+ (33, 1024, 1024),
24
+ (33, 8192, 128),
25
+ (64, 2048, 496),
26
+ (64, 16384, 1024),
27
+ (100, 8192, 496),
28
+ (128, 32768, 4096),
29
+ (256, 4096, 4096),
30
+ (512, 256, 1024),
31
+ (512, 8192, 4096),
32
+ (512, 16384, 128),
33
+ (512, 24576, 128),
34
+]
35
+
36
CUDA_DEVICES = [
37
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
38
]
@@ -116,9 +138,7 @@ def cutlass_int8_gemm_helper(m: int,
138
(out, a, b, scale_a, scale_b, bias))
139
140
119
-@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 100, 33])
120
-@pytest.mark.parametrize("n", [2048, 4096, 8192, 16384, 24576, 256, 1024])
121
-@pytest.mark.parametrize("k", [128, 496, 1024])
141
+@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
142
@pytest.mark.parametrize("per_act_token", [True, False])
143
@pytest.mark.parametrize("per_out_ch", [True, False])
144
@pytest.mark.parametrize("use_bias", [True, False])
@@ -129,9 +149,7 @@ def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
149
cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
150
151
132
-@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 33, 1])
133
-@pytest.mark.parametrize("n", [2048, 8192, 16384, 256, 1024])
134
-@pytest.mark.parametrize("k", [128, 496, 1024])
152
+@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
153
@pytest.mark.parametrize("per_act_token", [True, False])
154
@pytest.mark.parametrize("per_out_ch", [True, False])
155
@pytest.mark.parametrize("use_bias", [True, False])
tests/kernels/test_int8_quant.py
+3
-4
@@ -7,11 +7,10 @@ from vllm._custom_ops import scaled_int8_quant
7
from vllm.platforms import current_platform
8
9
DTYPES = [torch.half, torch.bfloat16, torch.float]
10
-HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
11
- 8193] # Arbitrary values for testing
10
+HIDDEN_SIZES = [16, 67, 768, 5137, 8193] # Arbitrary values for testing
11
NUM_TOKENS = [1, 7, 83, 4096] # Arbitrary values for testing
12
SEEDS = [0]
14
-SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
13
+SCALE = [0.1, 2.1]
14
15
16
def opcheck_int8_quant_static(output, input, scale, azp=None):
@@ -132,7 +131,7 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
131
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
132
@pytest.mark.parametrize("dtype", DTYPES)
133
@pytest.mark.parametrize("seed", SEEDS)
135
-@pytest.mark.parametrize("scale", SCALE[2:]) # Reduce test time
134
+@pytest.mark.parametrize("scale", SCALE)
135
@pytest.mark.parametrize("azp", [-255, 54])
136
@torch.inference_mode()
137
def test_static_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
tests/kernels/test_marlin_gemm.py
+1
-1
@@ -35,7 +35,7 @@ K_FULL_OPTS = [False, True]
35
USE_FP32_REDUCE_OPTS = [False, True]
36
37
MARLIN_K_CHUNKS = [128]
38
-MARLIN_N_CHUNKS = [64, 128, 256]
38
+MARLIN_N_CHUNKS = [64, 256]
39
40
MARLIN_24_K_CHUNKS = [128]
41
MARLIN_24_N_CHUNKS = [512]
tests/kernels/test_moe.py
+13
-10
@@ -20,12 +20,15 @@ from vllm.model_executor.models.mixtral import MixtralMoE
20
from vllm.platforms import current_platform
21
from vllm.scalar_type import scalar_types
22
23
+NUM_EXPERTS = [8, 64]
24
+TOP_KS = [2, 6]
25
24
-@pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
25
-@pytest.mark.parametrize("n", [2048, 256, 1024])
26
+
27
+@pytest.mark.parametrize("m", [1, 33, 64, 222, 1024 * 128])
28
+@pytest.mark.parametrize("n", [128, 1024, 2048])
29
@pytest.mark.parametrize("k", [128, 511, 1024])
27
-@pytest.mark.parametrize("e", [8, 64])
28
-@pytest.mark.parametrize("topk", [2, 6])
30
+@pytest.mark.parametrize("e", NUM_EXPERTS)
31
+@pytest.mark.parametrize("topk", TOP_KS)
32
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
33
def test_fused_moe(
34
m: int,
@@ -93,12 +96,12 @@ def test_mixtral_moe(dtype: torch.dtype):
96
atol=mixtral_moe_tol[dtype])
97
98
96
-@pytest.mark.parametrize("m", [64, 512, 222, 33, 1])
97
-@pytest.mark.parametrize("n", [128, 2048, 256, 1024])
98
-@pytest.mark.parametrize("k", [128, 1024, 512])
99
-@pytest.mark.parametrize("e", [8, 64])
100
-@pytest.mark.parametrize("topk", [2, 6])
101
-@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
99
+@pytest.mark.parametrize("m", [1, 33, 64, 222])
100
+@pytest.mark.parametrize("n", [128, 2048])
101
+@pytest.mark.parametrize("k", [128, 1024])
102
+@pytest.mark.parametrize("e", NUM_EXPERTS)
103
+@pytest.mark.parametrize("topk", TOP_KS)
104
+@pytest.mark.parametrize("group_size", [-1, 32, 128])
105
@pytest.mark.parametrize("act_order", [True, False])
106
@pytest.mark.parametrize("num_bits", [4, 8])
107
@pytest.mark.parametrize("is_k_full", [True, False])
tests/kernels/test_pos_encoding.py
+3
-3
@@ -11,10 +11,10 @@ from .allclose_default import get_default_atol, get_default_rtol
11
12
IS_NEOX_STYLE = [True, False]
13
DTYPES = [torch.half, torch.bfloat16, torch.float]
14
-HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
14
+HEAD_SIZES = [64, 80, 112, 120, 256]
15
ROTARY_DIMS = [None, 32] # None means rotary dim == head size
16
-NUM_HEADS = [7, 17] # Arbitrary values for testing
17
-BATCH_SIZES = [1, 5] # Arbitrary values for testing
16
+NUM_HEADS = [17] # Arbitrary values for testing
17
+BATCH_SIZES = [5] # Arbitrary values for testing
18
SEQ_LENS = [11, 8192] # Arbitrary values for testing
19
SEEDS = [0]
20
CUDA_DEVICES = [