6317a5174a · tduncan/Nvidia-Optimization-Stack-VLLM

.buildkite/lm-eval-harness/test_lm_eval_correctness.py

+1 -1

@@ -16,7 +16,7 @@ import numpy

16 import pytest

17 import yaml

18

19 -RTOL = 0.05

19 +RTOL = 0.08

20 TEST_DATA_FILE = os.environ.get(

21 "LM_EVAL_TEST_DATA_FILE",

22 ".buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct.yaml")

.buildkite/test-pipeline.yaml

+36 -5

@@ -317,15 +317,46 @@ steps:

317 commands:

318 - pytest -v -s compile/test_full_graph.py

319

320 -- label: Kernels Test %N # 1h each

321 - mirror_hardwares: [amd]

320 +- label: Kernels Core Operation Test

321 source_file_dependencies:

322 - csrc/

323 + - tests/kernels/core

324 + commands:

325 + - pytest -v -s kernels/core

326 +

327 +- label: Kernels Attention Test %N

328 + source_file_dependencies:

329 + - csrc/attention/

330 - vllm/attention

325 - - tests/kernels

331 + - vllm/v1/attention

332 + - tests/kernels/attention

333 commands:

327 - - pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

328 - parallelism: 4

334 + - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

335 + parallelism: 2

336 +

337 +- label: Kernels Quantization Test %N

338 + source_file_dependencies:

339 + - csrc/quantization/

340 + - vllm/model_executor/layers/quantization

341 + - tests/kernels/quantization

342 + commands:

343 + - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

344 + parallelism: 2

345 +

346 +- label: Kernels MoE Test

347 + source_file_dependencies:

348 + - csrc/moe/

349 + - tests/kernels/moe

350 + - vllm/model_executor/layers/fused_moe/

351 + commands:

352 + - pytest -v -s kernels/moe

353 +

354 +- label: Kernels Mamba Test

355 + source_file_dependencies:

356 + - csrc/mamba/

357 + - tests/kernels/mamba

358 + commands:

359 + - pytest -v -s kernels/mamba

360

361 - label: Tensorizer Test # 11min

362 # mirror_hardwares: [amd]

tests/kernels/attention/conftest.py renamed

tests/kernels/attention/test_attention.py renamed

+1 -2

@@ -6,13 +6,12 @@ from typing import Optional

6 import pytest

7 import torch

8

9 +from tests.kernels.allclose_default import get_default_atol, get_default_rtol

10 from tests.kernels.utils import opcheck

11 from vllm import _custom_ops as ops

12 from vllm.platforms import current_platform

13 from vllm.utils import get_max_shared_memory_bytes

14

14 -from .allclose_default import get_default_atol, get_default_rtol

15 -

15 if not current_platform.is_rocm():

16 from xformers import ops as xops

17 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask

tests/kernels/attention/test_attention_selector.py renamed

+9

@@ -156,6 +156,15 @@ def test_env(

156 expected = ("TRITON_MLA_VLLM_V1"

157 if use_v1 else "TRITON_MLA")

158 assert backend.get_name() == expected

159 + elif name == "FLASHINFER":

160 + backend = get_attn_backend(16,

161 + torch.float16,

162 + torch.float16,

163 + block_size,

164 + False,

165 + use_mla=use_mla)

166 + expected = "FLASHINFER_VLLM_V1" if use_v1 else name

167 + assert backend.get_name() == expected

168 else:

169 backend = get_attn_backend(16,

170 torch.float16,

tests/kernels/attention/test_blocksparse_attention.py renamed

+1 -2

@@ -6,14 +6,13 @@ from typing import Optional

6 import pytest

7 import torch

8

9 +from tests.kernels.allclose_default import get_default_atol, get_default_rtol

10 from vllm import _custom_ops as ops

11 from vllm.attention.ops.blocksparse_attention.interface import (

12 LocalStridedBlockSparseAttn)

13 from vllm.platforms import current_platform

14 from vllm.utils import get_max_shared_memory_bytes

15

15 -from .allclose_default import get_default_atol, get_default_rtol

16 -

16 FLOAT32_BYTES = torch.finfo(torch.float).bits // 8

17 # This will change depending on the compute capability.

18 # - 512 as a buffer

tests/kernels/attention/test_cache.py renamed

tests/kernels/attention/test_cascade_flash_attn.py renamed

tests/kernels/attention/test_encoder_decoder_attn.py renamed

tests/kernels/attention/test_flash_attn.py renamed

tests/kernels/attention/test_flashinfer.py renamed

tests/kernels/attention/test_flashmla.py renamed

tests/kernels/attention/test_lightning_attn.py renamed

tests/kernels/attention/test_merge_attn_states.py renamed

tests/kernels/attention/test_mha_attn.py renamed

tests/kernels/attention/test_mla_decode_cpu.py renamed

tests/kernels/attention/test_prefix_prefill.py renamed

tests/kernels/attention/test_rocm_attention_selector.py renamed

tests/kernels/attention/test_triton_decode_attention.py renamed

tests/kernels/core/test_activation.py renamed

+1 -2

@@ -5,6 +5,7 @@ import random

5 import pytest

6 import torch

7

8 +from tests.kernels.allclose_default import get_default_atol, get_default_rtol

9 from tests.kernels.utils import opcheck

10 from vllm.model_executor.layers.activation import (FastGELU, FatreluAndMul,

11 GeluAndMul, MulAndSilu,

@@ -12,8 +13,6 @@ from vllm.model_executor.layers.activation import (FastGELU, FatreluAndMul,

13 SiluAndMul)

14 from vllm.platforms import current_platform

15

15 -from .allclose_default import get_default_atol, get_default_rtol

16 -

16 DTYPES = [torch.half, torch.bfloat16, torch.float]

17 NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing

18 D = [512, 13824] # Arbitrary values for testing

tests/kernels/core/test_fused_quant_layernorm.py renamed

tests/kernels/core/test_layernorm.py renamed

tests/kernels/core/test_opcheck.py new

+25

@@ -0,0 +1,25 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +"""

3 +Tests for miscellaneous utilities

4 +"""

5 +

6 +import torch

7 +

8 +from tests.kernels.utils import opcheck

9 +

10 +

11 +def test_convert_fp8_opcheck():

12 + data = torch.randn((256, 256), dtype=torch.float32, device="cuda")

13 + result = torch.empty_like(data, dtype=torch.float8_e4m3fn)

14 + opcheck(torch.ops._C_cache_ops.convert_fp8, (result, data, 1.0, "fp8"))

15 +

16 +

17 +# TODO: Add this back, currently fails with

18 +# csrc/cuda_utils_kernels.cu:15 'invalid argument'

19 +# @pytest.mark.skipif(not current_platform.is_cuda(),

20 +# reason="Only supported for CUDA")

21 +# def test_cuda_utils_opcheck():

22 +# opcheck(torch.ops._C_cuda_utils.get_device_attribute, (0, 0))

23 +# opcheck(

24 +# torch.ops._C_cuda_utils.

25 +# get_max_shared_memory_per_block_device_attribute, (0, ))

tests/kernels/core/test_permute_cols.py renamed

tests/kernels/core/test_pos_encoding.py renamed

+1 -2

@@ -6,11 +6,10 @@ from typing import Callable, Optional

6 import pytest

7 import torch

8

9 +from tests.kernels.allclose_default import get_default_atol, get_default_rtol

10 from vllm.model_executor.layers.rotary_embedding import get_rope

11 from vllm.platforms import current_platform

12

12 -from .allclose_default import get_default_atol, get_default_rtol

13 -

13 IS_NEOX_STYLE = [True, False]

14 DTYPES = [torch.half, torch.bfloat16, torch.float]

15 HEAD_SIZES = [64, 80, 112, 120, 256]

tests/kernels/core/test_rotary_embedding.py renamed

tests/kernels/core/test_uva.py renamed

tests/kernels/mamba/test_causal_conv1d.py renamed

tests/kernels/mamba/test_mamba_mixer2.py renamed

tests/kernels/mamba/test_mamba_ssm.py renamed

tests/kernels/mamba/test_mamba_ssm_ssd.py renamed

tests/kernels/moe/test_cutlass_moe.py renamed

tests/kernels/moe/test_moe.py renamed

tests/kernels/moe/test_triton_moe_ptpc_fp8.py renamed

tests/kernels/quantization/test_allspark_gemm.py renamed

tests/kernels/quantization/test_aqlm.py renamed

tests/kernels/quantization/test_awq.py renamed

tests/kernels/quantization/test_awq_marlin.py renamed

tests/kernels/quantization/test_awq_triton.py renamed

tests/kernels/quantization/test_block_fp8.py renamed

+1 -2

@@ -6,6 +6,7 @@ import itertools

6 import pytest

7 import torch

8

9 +from tests.kernels.utils_block import native_w8a8_block_matmul

10 from vllm.config import VllmConfig, set_current_vllm_config

11 from vllm.model_executor.layers.activation import SiluAndMul

12 from vllm.model_executor.layers.fused_moe import fused_moe

@@ -18,8 +19,6 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (

19 per_token_group_quant_fp8, w8a8_block_fp8_matmul)

20 from vllm.platforms import current_platform

21

21 -from .utils_block import native_w8a8_block_matmul

22 -

22 dg_available = False

23 try:

24 import deep_gemm

tests/kernels/quantization/test_block_int8.py renamed

+1 -2

@@ -6,6 +6,7 @@ import itertools

6 import pytest

7 import torch

8

9 +from tests.kernels.utils_block import native_w8a8_block_matmul

10 from vllm.config import VllmConfig, set_current_vllm_config

11 from vllm.model_executor.layers.activation import SiluAndMul

12 from vllm.model_executor.layers.fused_moe import fused_moe

@@ -13,8 +14,6 @@ from vllm.model_executor.layers.quantization.utils.int8_utils import (

14 w8a8_block_int8_matmul)

15 from vllm.platforms import current_platform

16

16 -from .utils_block import native_w8a8_block_matmul

17 -

17 if current_platform.get_device_capability() < (7, 0):

18 pytest.skip("INT8 Triton requires CUDA 7.0 or higher",

19 allow_module_level=True)

tests/kernels/quantization/test_cutlass_2of4_sparse.py renamed

+1 -2

@@ -7,13 +7,12 @@ Run `pytest tests/kernels/test_semi_structured.py`.

7 import pytest

8 import torch

9

10 +from tests.kernels.utils import baseline_scaled_mm, to_fp8, to_int8

11 from vllm import _custom_ops as ops

12 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (

13 sparse_cutlass_supported)

14 from vllm.platforms import current_platform

15

15 -from .utils import baseline_scaled_mm, to_fp8, to_int8

16 -

16 CUDA_DEVICES = [

17 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)

18 ]

tests/kernels/quantization/test_cutlass_scaled_mm.py renamed

+1 -3

@@ -8,13 +8,11 @@ import random

8 import pytest

9 import torch

10

11 -from tests.kernels.utils import opcheck

11 +from tests.kernels.utils import baseline_scaled_mm, opcheck, to_fp8, to_int8

12 from vllm import _custom_ops as ops

13 from vllm.platforms import current_platform

14 from vllm.utils import cdiv

15

16 -from .utils import baseline_scaled_mm, to_fp8, to_int8

17 -

16 MNK_FACTORS = [

17 (1, 256, 128),

18 (1, 16384, 1024),

tests/kernels/quantization/test_fp8_quant.py renamed

tests/kernels/quantization/test_ggml.py renamed

tests/kernels/quantization/test_gguf.py renamed

tests/kernels/quantization/test_gptq.py renamed

tests/kernels/quantization/test_int8_kernel.py renamed

tests/kernels/quantization/test_int8_quant.py renamed

tests/kernels/quantization/test_machete_mm.py renamed

tests/kernels/quantization/test_marlin_gemm.py renamed

tests/kernels/quantization/test_nvfp4_quant.py renamed

tests/kernels/quantization/test_nvfp4_scaled_mm.py renamed

tests/kernels/quantization/test_triton_scaled_mm.py renamed

tests/kernels/test_utils.py deleted

-25

@@ -1,25 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -"""

3 -Tests for miscellaneous utilities

4 -"""

5 -

6 -import pytest

7 -import torch

8 -

9 -from tests.kernels.utils import opcheck

10 -from vllm.platforms import current_platform

11 -

12 -

13 -def test_convert_fp8_opcheck():

14 - data = torch.randn((256, 256), dtype=torch.float32, device="cuda")

15 - result = torch.empty_like(data, dtype=torch.float8_e4m3fn)

16 - opcheck(torch.ops._C_cache_ops.convert_fp8, (result, data, 1.0, "fp8"))

17 -

18 -

19 -@pytest.mark.skipif(not current_platform.is_cuda(),

20 - reason="Only supported for CUDA")

21 -def test_cuda_utils_opcheck():

22 - opcheck(torch.ops._C_cuda_utils.get_device_attribute, (0, 0))

23 - opcheck(

24 - torch.ops._C_cuda_utils.

25 - get_max_shared_memory_per_block_device_attribute, (0, ))

Categorize `tests/kernels/` based on kernel type (#16799)