@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2c8b9182b5

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 23:30 UTC

[CI] Reorganize compile tests so new tests are automatically included in CI (#28625)

Signed-off-by: Yanan Cao <gmagogsfm@gmail.com>

Yanan Cao committed Nov 19, 2025 at 06:13 UTC 2c8b9182b5ced00d83bed15ef8bc0ac6e079b6ee

17 files changed +74 -72

.buildkite/test-amd.yaml

+27 -30

@@ -187,7 +187,7 @@ steps:

187 - tests/distributed/test_utils

188 - tests/distributed/test_pynccl

189 - tests/distributed/test_events

190 - - tests/compile/test_basic_correctness

190 + - tests/compile/fullgraph/test_basic_correctness.py

191 - examples/offline_inference/rlhf.py

192 - examples/offline_inference/rlhf_colocate.py

193 - tests/examples/offline_inference/data_parallel.py

@@ -215,7 +215,7 @@ steps:

215 - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py

216 - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp

217 - pytest -v -s distributed/test_utils.py

218 - - pytest -v -s compile/test_basic_correctness.py

218 + - pytest -v -s compile/fullgraph/test_basic_correctness.py

219 - pytest -v -s distributed/test_pynccl.py

220 - pytest -v -s distributed/test_events.py

221 - pytest -v -s distributed/test_symm_mem_allreduce.py

@@ -493,17 +493,12 @@ steps:

493 - vllm/

494 - tests/compile

495 commands:

496 - - pytest -v -s compile/test_pass_manager.py

497 - - pytest -v -s compile/test_fusion.py

498 - - pytest -v -s compile/test_fusion_attn.py

499 - - pytest -v -s compile/test_functionalization.py

500 - - pytest -v -s compile/test_silu_mul_quant_fusion.py

501 - # - pytest -v -s compile/test_sequence_parallelism.py

502 - # - pytest -v -s compile/test_async_tp.py

503 - - pytest -v -s compile/test_fusion_all_reduce.py

504 - - pytest -v -s compile/test_decorator.py

505 - - pytest -v -s compile/test_noop_elimination.py

506 - - pytest -v -s compile/test_aot_compile.py

496 + # Run unit tests defined directly under compile/,

497 + # not including subdirectories, which are usually heavier

498 + # tests covered elsewhere.

499 + # Use `find` to launch multiple instances of pytest so that

500 + # they do not suffer from https://github.com/vllm-project/vllm/issues/28965

501 + - "find compile/ -maxdepth 1 -name 'test_*.py' -exec pytest -s -v {} \\\\;"

502

503 - label: PyTorch Fullgraph Smoke Test # 15min

504 timeout_in_minutes: 30

@@ -515,9 +510,11 @@ steps:

510 - vllm/

511 - tests/compile

512 commands:

518 - - pytest -v -s compile/test_basic_correctness.py

519 - - pytest -v -s compile/test_multimodal_compile.py

520 - - pytest -v -s compile/piecewise/

513 + # Run smoke tests under fullgraph directory, except test_full_graph.py

514 + # as it is a heavy test that is covered in other steps.

515 + # Use `find` to launch multiple instances of pytest so that

516 + # they do not suffer from https://github.com/vllm-project/vllm/issues/28965

517 + - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -exec pytest -s -v {} \\\\;"

518

519 - label: PyTorch Fullgraph Test # 27min

520 timeout_in_minutes: 40

@@ -529,10 +526,10 @@ steps:

526 - vllm/

527 - tests/compile

528 commands:

532 - - pytest -v -s compile/test_full_graph.py -k 'not test_fp8_kv_scale_compile'

529 + - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'

530 # Limit to no custom ops to reduce running time

531 # Wrap with quotes to escape yaml and avoid starting -k string with a -

535 - - "pytest -v -s compile/test_fusions_e2e.py -k 'TRITON and -quant_fp8'"

532 + - "pytest -v -s compile/distributed/test_fusions_e2e.py -k 'TRITON and -quant_fp8'"

533

534 - label: Cudagraph test

535 timeout_in_minutes: 20

@@ -1066,10 +1063,10 @@ steps:

1063 - pytest -v -s tests/compile/test_fusion_attn.py

1064 - pytest -v -s tests/compile/test_silu_mul_quant_fusion.py

1065 # this runner has 2 GPUs available even though num_gpus=2 is not set

1069 - - pytest -v -s tests/compile/test_fusion_all_reduce.py

1066 + - pytest -v -s tests/compile/distributed/test_fusion_all_reduce.py

1067 # Limit to Inductor partition, no custom ops, and allreduce & attn fusion to reduce running time

1068 # Wrap with quotes to escape yaml

1072 - - "pytest -v -s tests/compile/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm -k 'True and Llama-3.1 and -quant_fp8 and -rms_norm'"

1069 + - "pytest -v -s tests/compile/distributed/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm -k 'True and Llama-3.1 and -quant_fp8 and -rms_norm'"

1070

1071 - label: Blackwell Fusion E2E Tests # 30 min

1072 timeout_in_minutes: 40

@@ -1086,14 +1083,14 @@ steps:

1083 - vllm/model_executor/layers/layernorm.py

1084 - vllm/model_executor/layers/activation.py

1085 - vllm/model_executor/layers/quantization/input_quant_fp8.py

1089 - - tests/compile/test_fusions_e2e.py

1090 - - tests/compile/test_full_graph.py

1086 + - tests/compile/distributed/test_fusions_e2e.py

1087 + - tests/compile/fullgraph/test_full_graph.py

1088 commands:

1089 - nvidia-smi

1090 # Run all e2e fusion tests

1094 - - pytest -v -s tests/compile/test_fusions_e2e.py

1091 + - pytest -v -s tests/compile/distributed/test_fusions_e2e.py

1092 # test_fp8_kv_scale_compile requires FlashAttention (not supported on default L4/L40)

1096 - - pytest -v -s tests/compile/test_full_graph.py::test_fp8_kv_scale_compile

1093 + - pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile

1094

1095 - label: ROCm GPT-OSS Eval

1096 timeout_in_minutes: 60

@@ -1198,7 +1195,7 @@ steps:

1195 - vllm/worker/worker_base.py

1196 - vllm/v1/engine/

1197 - vllm/v1/worker/

1201 - - tests/compile/test_basic_correctness.py

1198 + - tests/compile/fullgraph/test_basic_correctness.py

1199 - tests/compile/test_wrapper.py

1200 - tests/distributed/

1201 - tests/entrypoints/llm/test_collective_rpc.py

@@ -1211,7 +1208,7 @@ steps:

1208 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py

1209 - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py

1210 - pytest -v -s entrypoints/llm/test_collective_rpc.py

1214 - - pytest -v -s ./compile/test_basic_correctness.py

1211 + - pytest -v -s ./compile/fullgraph/test_basic_correctness.py

1212 - pytest -v -s ./compile/test_wrapper.py

1213 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'

1214 - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'

@@ -1417,10 +1414,10 @@ steps:

1414 working_dir: "/vllm-workspace/"

1415 num_gpus: 2

1416 commands:

1420 - - pytest -v -s tests/compile/test_async_tp.py

1421 - - pytest -v -s tests/compile/test_sequence_parallelism.py

1422 - - pytest -v -s tests/compile/test_fusion_all_reduce.py

1423 - - pytest -v -s tests/compile/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm

1417 + - pytest -v -s tests/compile/distributed/test_async_tp.py

1418 + - pytest -v -s tests/compile/distributed/test_sequence_parallelism.py

1419 + - pytest -v -s tests/compile/distributed/test_fusion_all_reduce.py

1420 + - pytest -v -s tests/compile/distributed/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm

1421 - pytest -v -s tests/distributed/test_context_parallel.py

1422 - CUDA_VISIBLE_DEVICES=1,2 VLLM_ALL2ALL_BACKEND=deepep_high_throughput VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model Qwen/Qwen1.5-MoE-A2.7B --tp-size=1 --dp-size=2 --max-model-len 2048

1423 - pytest -v -s tests/v1/distributed/test_dbo.py

.buildkite/test-pipeline.yaml

+31 -31

@@ -167,7 +167,7 @@ steps:

167 - tests/distributed/test_utils

168 - tests/distributed/test_pynccl

169 - tests/distributed/test_events

170 - - tests/compile/test_basic_correctness

170 + - tests/compile/fullgraph/test_basic_correctness.py

171 - examples/offline_inference/rlhf.py

172 - examples/offline_inference/rlhf_colocate.py

173 - tests/examples/offline_inference/data_parallel.py

@@ -197,7 +197,7 @@ steps:

197 - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py

198 - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp

199 - pytest -v -s distributed/test_utils.py

200 - - pytest -v -s compile/test_basic_correctness.py

200 + - pytest -v -s compile/fullgraph/test_basic_correctness.py

201 - pytest -v -s distributed/test_pynccl.py

202 - pytest -v -s distributed/test_events.py

203 - pytest -v -s distributed/test_symm_mem_allreduce.py

@@ -445,18 +445,12 @@ steps:

445 - vllm/

446 - tests/compile

447 commands:

448 - - pytest -v -s compile/test_graph_partition.py

449 - - pytest -v -s compile/test_config.py

450 - - pytest -v -s compile/test_pass_manager.py

451 - - pytest -v -s compile/test_fusion.py

452 - - pytest -v -s compile/test_fusion_attn.py

453 - - pytest -v -s compile/test_functionalization.py

454 - - pytest -v -s compile/test_silu_mul_quant_fusion.py

455 - - pytest -v -s compile/test_fusion_all_reduce.py

456 - - pytest -v -s compile/test_decorator.py

457 - - pytest -v -s compile/test_noop_elimination.py

458 - - pytest -v -s compile/test_aot_compile.py

459 - - pytest -v -s compile/test_qk_norm_rope_fusion.py

448 + # Run unit tests defined directly under compile/,

449 + # not including subdirectories, which are usually heavier

450 + # tests covered elsewhere.

451 + # Use `find` to launch multiple instances of pytest so that

452 + # they do not suffer from https://github.com/vllm-project/vllm/issues/28965

453 + - "find compile/ -maxdepth 1 -name 'test_*.py' -exec pytest -s -v {} \\\\;"

454

455 - label: PyTorch Fullgraph Smoke Test # 15min

456 timeout_in_minutes: 30

@@ -466,9 +460,11 @@ steps:

460 - vllm/

461 - tests/compile

462 commands:

469 - - pytest -v -s compile/test_basic_correctness.py

470 - - pytest -v -s compile/test_multimodal_compile.py

471 - - pytest -v -s compile/piecewise/

463 + # Run smoke tests under fullgraph directory, except test_full_graph.py

464 + # as it is a heavy test that is covered in other steps.

465 + # Use `find` to launch multiple instances of pytest so that

466 + # they do not suffer from https://github.com/vllm-project/vllm/issues/28965

467 + - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -exec pytest -s -v {} \\\\;"

468

469 - label: PyTorch Fullgraph Test # 27min

470 timeout_in_minutes: 40

@@ -479,10 +475,10 @@ steps:

475 - tests/compile

476 commands:

477 # fp8 kv scales not supported on sm89, tested on Blackwell instead

482 - - pytest -v -s compile/test_full_graph.py -k 'not test_fp8_kv_scale_compile'

478 + - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'

479 # Limit to no custom ops to reduce running time

480 # Wrap with quotes to escape yaml and avoid starting -k string with a -

485 - - "pytest -v -s compile/test_fusions_e2e.py -k 'TRITON and not +quant_fp8 and not Llama-4'"

481 + - "pytest -v -s compile/distributed/test_fusions_e2e.py -k 'TRITON and not +quant_fp8 and not Llama-4'"

482

483 - label: Cudagraph test

484 timeout_in_minutes: 20

@@ -939,17 +935,22 @@ steps:

935 - vllm/model_executor/layers/layernorm.py

936 - vllm/model_executor/layers/activation.py

937 - vllm/model_executor/layers/quantization/input_quant_fp8.py

938 + - tests/compile/test_fusion_attn.py

939 + - tests/compile/test_silu_mul_quant_fusion.py

940 + - tests/compile/distributed/test_fusion_all_reduce.py

941 + - tests/compile/distributed/test_fusions_e2e.py

942 + - tests/compile/fullgraph/test_full_graph.py

943 commands:

944 - nvidia-smi

945 - pytest -v -s tests/compile/test_fusion_attn.py

946 - pytest -v -s tests/compile/test_silu_mul_quant_fusion.py

947 # this runner has 2 GPUs available even though num_gpus=2 is not set

947 - - pytest -v -s tests/compile/test_fusion_all_reduce.py

948 + - pytest -v -s tests/compile/distributed/test_fusion_all_reduce.py

949 # Limit to Inductor partition, no custom ops, and allreduce & attn fusion to reduce running time

950 # Wrap with quotes to escape yaml

950 - - "pytest -v -s tests/compile/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm -k 'True and not +quant_fp8 and not +rms_norm'"

951 + - "pytest -v -s tests/compile/distributed/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm -k 'True and not +quant_fp8 and not +rms_norm'"

952 # test_fp8_kv_scale_compile requires FlashAttention (not supported on default L4/L40)

952 - - pytest -v -s tests/compile/test_full_graph.py::test_fp8_kv_scale_compile

953 + - pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile

954

955 - label: Blackwell Fusion E2E Tests # 30 min

956 timeout_in_minutes: 40

@@ -966,12 +967,11 @@ steps:

967 - vllm/model_executor/layers/layernorm.py

968 - vllm/model_executor/layers/activation.py

969 - vllm/model_executor/layers/quantization/input_quant_fp8.py

969 - - tests/compile/test_fusions_e2e.py

970 - - tests/compile/test_full_graph.py

970 + - tests/compile/distributed/test_fusions_e2e.py

971 commands:

972 - nvidia-smi

973 # Run all e2e fusion tests

974 - - pytest -v -s tests/compile/test_fusions_e2e.py

974 + - pytest -v -s tests/compile/distributed/test_fusions_e2e.py

975

976 - label: Blackwell GPT-OSS Eval

977 timeout_in_minutes: 60

@@ -1069,7 +1069,7 @@ steps:

1069 - vllm/worker/worker_base.py

1070 - vllm/v1/engine/

1071 - vllm/v1/worker/

1072 - - tests/compile/test_basic_correctness.py

1072 + - tests/compile/fullgraph/test_basic_correctness.py

1073 - tests/compile/test_wrapper.py

1074 - tests/distributed/

1075 - tests/entrypoints/llm/test_collective_rpc.py

@@ -1084,7 +1084,7 @@ steps:

1084 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py

1085 - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py

1086 - pytest -v -s entrypoints/llm/test_collective_rpc.py

1087 - - pytest -v -s ./compile/test_basic_correctness.py

1087 + - pytest -v -s ./compile/fullgraph/test_basic_correctness.py

1088 - pytest -v -s ./compile/test_wrapper.py

1089 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'

1090 - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'

@@ -1264,10 +1264,10 @@ steps:

1264 working_dir: "/vllm-workspace/"

1265 num_gpus: 2

1266 commands:

1267 - - pytest -v -s tests/compile/test_async_tp.py

1268 - - pytest -v -s tests/compile/test_sequence_parallelism.py

1269 - - pytest -v -s tests/compile/test_fusion_all_reduce.py

1270 - - "pytest -v -s tests/compile/test_fusions_e2e.py -k 'not Llama-4'"

1267 + - pytest -v -s tests/compile/distributed/test_async_tp.py

1268 + - pytest -v -s tests/compile/distributed/test_sequence_parallelism.py

1269 + - pytest -v -s tests/compile/distributed/test_fusion_all_reduce.py

1270 + - "pytest -v -s tests/compile/distributed/test_fusions_e2e.py -k 'not Llama-4'"

1271 - pytest -v -s tests/distributed/test_sequence_parallel.py

1272 - pytest -v -s tests/distributed/test_context_parallel.py

1273 - CUDA_VISIBLE_DEVICES=1,2 VLLM_ALL2ALL_BACKEND=deepep_high_throughput VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model Qwen/Qwen1.5-MoE-A2.7B --tp-size=1 --dp-size=2 --max-model-len 2048

tests/compile/README.md new

@@ -0,0 +1,5 @@

1 +# compile test folder structure

2 +

3 +- `compile/test_*.py` : various unit tests meant for testing particular code path/features. Future tests are most likely added here. New test files added here will be included in CI automatically

4 +- `compile/fullgraph/` : full model tests, including all tests previously in compile/piecewise. These tests do not target particular features. New test files added here will be included in CI automatically

5 +- `compile/distributed/` : tests that require multiple GPUs. New test files added here will **NOT** be included in CI automatically as these tests generally need to be manually configured to run in runners with particular number/type of GPUs.

tests/compile/distributed/__init__.py renamed

tests/compile/distributed/test_async_tp.py renamed

+3 -3

@@ -27,13 +27,13 @@ from vllm.distributed.parallel_state import (

27 from vllm.platforms import current_platform

28 from vllm.utils.system_utils import update_environment_variables

30 -from ..models.registry import HF_EXAMPLE_MODELS

31 -from ..utils import (

30 +from ...models.registry import HF_EXAMPLE_MODELS

31 +from ...utils import (

32 compare_two_settings,

33 create_new_process_for_each_test,

34 multi_gpu_test,

35 )

36 -from .backend import TestBackend

36 +from ..backend import TestBackend

38 FP8_DTYPE = current_platform.fp8_dtype()

tests/compile/distributed/test_fusion_all_reduce.py renamed

+2 -2

@@ -33,8 +33,8 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (

33 from vllm.platforms import current_platform

34 from vllm.utils.system_utils import update_environment_variables

36 -from ..utils import has_module_attribute, multi_gpu_test

37 -from .backend import TestBackend

36 +from ...utils import has_module_attribute, multi_gpu_test

37 +from ..backend import TestBackend

40 class TestAllReduceRMSNormModel(torch.nn.Module):

tests/compile/distributed/test_fusions_e2e.py renamed

+1 -1

@@ -18,7 +18,7 @@ from vllm.platforms import current_platform

18 from vllm.utils.flashinfer import has_flashinfer

19 from vllm.utils.torch_utils import is_torch_equal_or_newer

21 -from ..utils import flat_product, multi_gpu_test

21 +from ...utils import flat_product, multi_gpu_test

23 is_blackwell = lambda: current_platform.is_device_capability(100)

24 """Are we running on Blackwell, a lot of tests depend on it"""

tests/compile/distributed/test_sequence_parallelism.py renamed

+2 -2

@@ -32,8 +32,8 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import Fp8LinearOp

32 from vllm.platforms import current_platform

33 from vllm.utils.system_utils import update_environment_variables

35 -from ..utils import multi_gpu_test

36 -from .backend import TestBackend

35 +from ...utils import multi_gpu_test

36 +from ..backend import TestBackend

38 FP8_DTYPE = current_platform.fp8_dtype()

39 prompts = [

tests/compile/fullgraph/__init__.py

tests/compile/fullgraph/test_basic_correctness.py renamed

+1 -1

@@ -7,7 +7,7 @@ import pytest

7 from vllm.config import CompilationMode

8 from vllm.utils.torch_utils import cuda_device_count_stateless

10 -from ..utils import compare_all_settings

10 +from ...utils import compare_all_settings

13 @dataclasses.dataclass

tests/compile/fullgraph/test_full_cudagraph.py renamed

tests/compile/fullgraph/test_full_graph.py renamed

+1 -1

@@ -15,7 +15,7 @@ from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode, PassC

15 from vllm.platforms import current_platform

16 from vllm.utils.torch_utils import is_torch_equal_or_newer

18 -from ..utils import create_new_process_for_each_test

18 +from ...utils import create_new_process_for_each_test

21 def models_list(*, all: bool = True, keywords: list[str] | None = None):

tests/compile/fullgraph/test_multimodal_compile.py renamed

tests/compile/fullgraph/test_multiple_graphs.py renamed

tests/compile/fullgraph/test_simple.py renamed

tests/compile/fullgraph/test_toy_llama.py renamed

vllm/env_override.py

+1 -1

@@ -95,7 +95,7 @@ def memory_plan_reuse_patched(self):

95 # ===================================================

96 # This change monkeypatches get_graph_partition_signature in pytorch 2.9.0 to

97 # fix inductor partition + attention-nvfp4 quant fusion, tested in

98 -# `tests/compile/test_fusions_e2e.py::test_attn_quant`.

98 +# `tests/compile/distributed/test_fusions_e2e.py::test_attn_quant`.

99 # For more context, see https://github.com/pytorch/pytorch/pull/165815.

100

101