A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 18:15 UTC
[CI] Split Distributed Tests (4 GPUs) into 3 parallel jobs (#37015)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Kevin H. Luu committed
Mar 13, 2026 at 21:21 UTC
74fe80ee9594bbc6c0d0c979dbb9d56fae0e789b
1 file changed
+40
-20
.buildkite/test_areas/distributed.yaml
+40
-20
@@ -50,24 +50,18 @@ steps:
50
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
51
- pytest -v -s v1/worker/test_worker_memory_snapshot.py
52
53
-- label: Distributed Tests (4 GPUs)
54
- timeout_in_minutes: 50
53
+- label: Distributed Torchrun + Examples (4 GPUs)
54
+ timeout_in_minutes: 30
55
working_dir: "/vllm-workspace/tests"
56
num_devices: 4
57
source_file_dependencies:
58
- vllm/distributed/
59
- - tests/distributed/test_utils
60
- - tests/distributed/test_pynccl
61
- - tests/distributed/test_events
62
- - tests/compile/fullgraph/test_basic_correctness.py
59
+ - tests/distributed/test_torchrun_example.py
60
+ - tests/distributed/test_torchrun_example_moe.py
61
- examples/offline_inference/rlhf.py
62
- examples/offline_inference/rlhf_colocate.py
63
- examples/offline_inference/new_weight_syncing/
64
- tests/examples/offline_inference/data_parallel.py
67
- - tests/v1/distributed
68
- - tests/v1/engine/test_engine_core_client.py
69
- - tests/distributed/test_symm_mem_allreduce.py
70
- - tests/distributed/test_multiproc_executor.py
65
commands:
66
# https://github.com/NVIDIA/nccl/issues/1838
67
- export NCCL_CUMEM_HOST_ENABLE=0
@@ -85,6 +79,27 @@ steps:
79
- TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
80
# test with internal dp
81
- python3 ../examples/offline_inference/data_parallel.py --enforce-eager
82
+ # OLD rlhf examples
83
+ - cd ../examples/offline_inference
84
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf.py
85
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 RAY_DEDUP_LOGS=0 python3 rlhf_colocate.py
86
+ # NEW rlhf examples
87
+ - cd new_weight_syncing
88
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf_nccl.py
89
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf_ipc.py
90
+
91
+- label: Distributed DP Tests (4 GPUs)
92
+ timeout_in_minutes: 30
93
+ working_dir: "/vllm-workspace/tests"
94
+ num_devices: 4
95
+ source_file_dependencies:
96
+ - vllm/distributed/
97
+ - tests/v1/distributed
98
+ - tests/v1/engine/test_engine_core_client.py
99
+ - tests/distributed/test_utils
100
+ commands:
101
+ # https://github.com/NVIDIA/nccl/issues/1838
102
+ - export NCCL_CUMEM_HOST_ENABLE=0
103
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
104
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
105
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
@@ -92,22 +107,27 @@ steps:
107
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
108
- pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
109
- pytest -v -s distributed/test_utils.py
110
+
111
+- label: Distributed Compile + Comm (4 GPUs)
112
+ timeout_in_minutes: 30
113
+ working_dir: "/vllm-workspace/tests"
114
+ num_devices: 4
115
+ source_file_dependencies:
116
+ - vllm/distributed/
117
+ - tests/distributed/test_pynccl
118
+ - tests/distributed/test_events
119
+ - tests/compile/fullgraph/test_basic_correctness.py
120
+ - tests/distributed/test_symm_mem_allreduce.py
121
+ - tests/distributed/test_multiproc_executor.py
122
+ commands:
123
+ # https://github.com/NVIDIA/nccl/issues/1838
124
+ - export NCCL_CUMEM_HOST_ENABLE=0
125
- pytest -v -s compile/fullgraph/test_basic_correctness.py
126
- pytest -v -s distributed/test_pynccl.py
127
- pytest -v -s distributed/test_events.py
128
- pytest -v -s distributed/test_symm_mem_allreduce.py
129
# test multi-node TP with multiproc executor (simulated on single node)
130
- pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
101
- # TODO: create a dedicated test section for multi-GPU example tests
102
- # when we have multiple distributed example tests
103
- # OLD rlhf examples
104
- - cd ../examples/offline_inference
105
- - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf.py
106
- - VLLM_ALLOW_INSECURE_SERIALIZATION=1 RAY_DEDUP_LOGS=0 python3 rlhf_colocate.py
107
- # NEW rlhf examples
108
- - cd new_weight_syncing
109
- - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf_nccl.py
110
- - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf_ipc.py
131
132
- label: Distributed Tests (8 GPUs)(H100)
133
timeout_in_minutes: 10