| 1 | group: Distributed |
| 2 | depends_on: |
| 3 | - image-build |
| 4 | steps: |
| 5 | - label: ":nvidia: (L4) Distributed Comm Ops" |
| 6 | key: distributed-comm-ops |
| 7 | timeout_in_minutes: 25 |
| 8 | working_dir: "/vllm-workspace/tests" |
| 9 | device: l4 |
| 10 | num_devices: 2 |
| 11 | source_file_dependencies: |
| 12 | - vllm/distributed |
| 13 | - "!vllm/distributed/kv_transfer/" |
| 14 | - tests/distributed |
| 15 | commands: |
| 16 | - pytest -v -s distributed/test_comm_ops.py |
| 17 | - pytest -v -s distributed/test_shm_broadcast.py |
| 18 | - pytest -v -s distributed/test_shm_buffer.py |
| 19 | - pytest -v -s distributed/test_shm_storage.py |
| 20 | |
| 21 | - label: ":nvidia: (L4) Sharded RDT Weight Transfer" |
| 22 | key: sharded-rdt-weight-transfer |
| 23 | device: l4 |
| 24 | timeout_in_minutes: 20 |
| 25 | working_dir: "/vllm-workspace/tests" |
| 26 | num_devices: 1 |
| 27 | source_file_dependencies: |
| 28 | - vllm/distributed/weight_transfer/ |
| 29 | - tests/distributed/test_sharded_rdt_plan.py |
| 30 | - tests/distributed/test_sharded_rdt_producer.py |
| 31 | - tests/distributed/test_sharded_rdt_trainer.py |
| 32 | commands: |
| 33 | - pytest -v -s distributed/test_sharded_rdt_plan.py |
| 34 | - pytest -v -s distributed/test_sharded_rdt_producer.py |
| 35 | - pytest -v -s distributed/test_sharded_rdt_trainer.py |
| 36 | mirror: |
| 37 | amd: |
| 38 | label: ":amd: (MI355 DPX) Sharded RDT Weight Transfer" |
| 39 | dind: false |
| 40 | device: mi355_dpx |
| 41 | num_devices: 1 |
| 42 | timeout_in_minutes: 45 |
| 43 | working_dir: /vllm-workspace/tests |
| 44 | depends_on: |
| 45 | - image-build-amd |
| 46 | source_file_dependencies: |
| 47 | - vllm/distributed/weight_transfer/ |
| 48 | - tests/distributed/test_sharded_rdt_plan.py |
| 49 | - tests/distributed/test_sharded_rdt_producer.py |
| 50 | - tests/distributed/test_sharded_rdt_trainer.py |
| 51 | - vllm/platforms/rocm.py |
| 52 | env: |
| 53 | VLLM_WORKER_MULTIPROC_METHOD: "spawn" |
| 54 | commands: |
| 55 | - uv pip install --system --no-deps 'ray==2.56.1' |
| 56 | - pytest -v -s distributed/test_sharded_rdt_plan.py |
| 57 | - pytest -v -s distributed/test_sharded_rdt_producer.py |
| 58 | - pytest -v -s distributed/test_sharded_rdt_trainer.py |
| 59 | |
| 60 | - label: ":nvidia: (L4) Distributed DP Basic" |
| 61 | key: distributed-dp-tests-2-gpus |
| 62 | timeout_in_minutes: 45 |
| 63 | working_dir: "/vllm-workspace/tests" |
| 64 | device: l4 |
| 65 | num_devices: 2 |
| 66 | source_file_dependencies: |
| 67 | - vllm/distributed/ |
| 68 | - "!vllm/distributed/kv_transfer/" |
| 69 | - vllm/engine/ |
| 70 | - vllm/v1/executor/ |
| 71 | - vllm/worker/worker_base.py |
| 72 | - vllm/v1/engine/ |
| 73 | - vllm/v1/worker/ |
| 74 | - tests/v1/distributed |
| 75 | - tests/entrypoints/launchers/api_server/test_multi_api_servers.py |
| 76 | - tests/v1/e2e/general/test_sharded_sampling.py |
| 77 | - tests/v1/e2e/spec_decode/test_sharded_sampling.py |
| 78 | - tests/v1/e2e/spec_decode/utils.py |
| 79 | commands: |
| 80 | # https://github.com/NVIDIA/nccl/issues/1838 |
| 81 | - export NCCL_CUMEM_HOST_ENABLE=0 |
| 82 | - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py |
| 83 | - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py |
| 84 | - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py |
| 85 | - DP_SIZE=2 pytest -v -s entrypoints/launchers/api_server/test_multi_api_servers.py |
| 86 | # Batch-sharded sampling requires TP=2 |
| 87 | - pytest -v -s v1/e2e/general/test_sharded_sampling.py |
| 88 | - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py |
| 89 | mirror: |
| 90 | amd: |
| 91 | label: ":amd: (MI355) Distributed DP Basic" |
| 92 | dind: false |
| 93 | device: mi355_2 |
| 94 | timeout_in_minutes: 45 |
| 95 | depends_on: |
| 96 | - image-build-amd |
| 97 | source_file_dependencies: |
| 98 | - vllm/distributed/ |
| 99 | - "!vllm/distributed/kv_transfer/" |
| 100 | - vllm/engine/ |
| 101 | - vllm/v1/executor/ |
| 102 | - vllm/worker/worker_base.py |
| 103 | - vllm/v1/engine/ |
| 104 | - vllm/v1/worker/ |
| 105 | - tests/v1/distributed |
| 106 | - tests/entrypoints/launchers/api_server/test_multi_api_servers.py |
| 107 | - vllm/platforms/rocm.py |
| 108 | |
| 109 | - label: ":nvidia: (L4) Distributed Compile + RPC" |
| 110 | key: distributed-compile-rpc-tests-2-gpus |
| 111 | timeout_in_minutes: 65 |
| 112 | working_dir: "/vllm-workspace/tests" |
| 113 | device: l4 |
| 114 | num_devices: 2 |
| 115 | source_file_dependencies: |
| 116 | - vllm/compilation/ |
| 117 | - vllm/distributed/ |
| 118 | - "!vllm/distributed/kv_transfer/" |
| 119 | - vllm/engine/ |
| 120 | - vllm/v1/executor/ |
| 121 | - vllm/worker/worker_base.py |
| 122 | - vllm/v1/engine/ |
| 123 | - vllm/v1/worker/ |
| 124 | - tests/compile/test_wrapper.py |
| 125 | - tests/entrypoints/llm/test_collective_rpc.py |
| 126 | commands: |
| 127 | # https://github.com/NVIDIA/nccl/issues/1838 |
| 128 | - export NCCL_CUMEM_HOST_ENABLE=0 |
| 129 | - pytest -v -s entrypoints/llm/test_collective_rpc.py |
| 130 | - pytest -v -s ./compile/test_wrapper.py |
| 131 | mirror: |
| 132 | amd: |
| 133 | label: ":amd: (MI355) Distributed Compile + RPC" |
| 134 | dind: false |
| 135 | device: mi355_2 |
| 136 | timeout_in_minutes: 65 |
| 137 | depends_on: |
| 138 | - image-build-amd |
| 139 | source_file_dependencies: |
| 140 | - vllm/compilation/ |
| 141 | - vllm/distributed/ |
| 142 | - "!vllm/distributed/kv_transfer/" |
| 143 | - vllm/engine/ |
| 144 | - vllm/v1/executor/ |
| 145 | - vllm/worker/worker_base.py |
| 146 | - vllm/v1/engine/ |
| 147 | - vllm/v1/worker/ |
| 148 | - tests/compile/test_wrapper.py |
| 149 | - tests/entrypoints/llm/test_collective_rpc.py |
| 150 | - vllm/platforms/rocm.py |
| 151 | commands: |
| 152 | - pytest -v -s entrypoints/llm/test_collective_rpc.py |
| 153 | - pytest -v -s ./compile/test_wrapper.py |
| 154 | |
| 155 | - label: ":nvidia: (L4) Distributed Torchrun + Shutdown" |
| 156 | key: distributed-torchrun-shutdown-tests-2-gpus |
| 157 | timeout_in_minutes: 30 |
| 158 | working_dir: "/vllm-workspace/tests" |
| 159 | device: l4 |
| 160 | num_devices: 2 |
| 161 | source_file_dependencies: |
| 162 | - vllm/distributed/ |
| 163 | - "!vllm/distributed/kv_transfer/" |
| 164 | - vllm/engine/ |
| 165 | - vllm/v1/executor/ |
| 166 | - vllm/worker/worker_base.py |
| 167 | - vllm/v1/engine/ |
| 168 | - vllm/v1/worker/ |
| 169 | - tests/distributed/ |
| 170 | - tests/v1/shutdown |
| 171 | - tests/v1/worker/test_worker_memory_snapshot.py |
| 172 | commands: |
| 173 | # https://github.com/NVIDIA/nccl/issues/1838 |
| 174 | - export NCCL_CUMEM_HOST_ENABLE=0 |
| 175 | - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' |
| 176 | - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' |
| 177 | - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown |
| 178 | - pytest -v -s v1/worker/test_worker_memory_snapshot.py |
| 179 | |
| 180 | - label: ":nvidia: (L4) Distributed Torchrun + Examples" |
| 181 | key: distributed-torchrun-examples-4-gpus |
| 182 | timeout_in_minutes: 45 |
| 183 | working_dir: "/vllm-workspace" |
| 184 | device: l4 |
| 185 | num_devices: 4 |
| 186 | source_file_dependencies: |
| 187 | - vllm/distributed/ |
| 188 | - "!vllm/distributed/kv_transfer/" |
| 189 | - tests/distributed/test_torchrun_example.py |
| 190 | - tests/distributed/test_torchrun_example_moe.py |
| 191 | - examples/rl/ |
| 192 | - examples/features/data_parallel/data_parallel_offline.py |
| 193 | commands: |
| 194 | # https://github.com/NVIDIA/nccl/issues/1838 |
| 195 | - export NCCL_CUMEM_HOST_ENABLE=0 |
| 196 | # test with torchrun tp=2 and external_dp=2 |
| 197 | - torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py |
| 198 | # test with torchrun tp=2 and pp=2 |
| 199 | - PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py |
| 200 | # test with torchrun tp=4 and dp=1 |
| 201 | - TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py |
| 202 | # test with torchrun tp=2, pp=2 and dp=1 |
| 203 | - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py |
| 204 | # test with torchrun tp=1 and dp=4 with ep |
| 205 | - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py |
| 206 | # test with torchrun tp=2 and dp=2 with ep |
| 207 | - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py |
| 208 | # test with internal dp |
| 209 | - python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager |
| 210 | # rlhf examples (each launches its own `vllm serve` and tears it down) |
| 211 | - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py |
| 212 | - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py |
| 213 | # sharded RDT: 2 FSDP2 trainer ranks -> 2 vLLM DP ranks with EP. Its data |
| 214 | # plane is NIXL, which ships only in the kv-connector set, and the script |
| 215 | # also drops the nixl-cu wheel variant that does not match this image. Kept |
| 216 | # last so the commands above run in an unmodified environment. RDT_MODEL swaps |
| 217 | # the script's reference MoE for a tiny one: no weights are downloaded either |
| 218 | # way, but every trainer rank builds the model before sharding it, and the |
| 219 | # default does not fit an L4. |
| 220 | - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh |
| 221 | - RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py |
| 222 | mirror: |
| 223 | amd: |
| 224 | label: ":amd: (MI355) Distributed Torchrun + Examples" |
| 225 | dind: false |
| 226 | device: mi355_4 |
| 227 | timeout_in_minutes: 55 |
| 228 | depends_on: |
| 229 | - image-build-amd |
| 230 | source_file_dependencies: |
| 231 | - vllm/distributed/ |
| 232 | - "!vllm/distributed/kv_transfer/" |
| 233 | - tests/distributed/test_torchrun_example.py |
| 234 | - tests/distributed/test_torchrun_example_moe.py |
| 235 | - examples/rl/ |
| 236 | - examples/features/data_parallel/data_parallel_offline.py |
| 237 | - .buildkite/scripts/install-kv-connectors.sh |
| 238 | - requirements/kv_connectors_rocm.txt |
| 239 | - requirements/test/rocm.in |
| 240 | - requirements/test/rocm.txt |
| 241 | - vllm/platforms/rocm.py |
| 242 | commands: |
| 243 | # Sharded RDT needs Ray >= 2.56; keep the ROCm image's dependencies. |
| 244 | - uv pip install --system --no-deps 'ray==2.56.1' |
| 245 | - export NCCL_CUMEM_HOST_ENABLE=0 |
| 246 | - torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py |
| 247 | - PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py |
| 248 | - TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py |
| 249 | - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py |
| 250 | - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py |
| 251 | - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py |
| 252 | - python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager |
| 253 | - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py |
| 254 | - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py |
| 255 | - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh |
| 256 | - RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py |
| 257 | |
| 258 | - label: ":nvidia: (L4) Distributed DP Extended" |
| 259 | key: distributed-dp-tests-4-gpus |
| 260 | timeout_in_minutes: 45 |
| 261 | working_dir: "/vllm-workspace/tests" |
| 262 | device: l4 |
| 263 | num_devices: 4 |
| 264 | source_file_dependencies: |
| 265 | - vllm/distributed/ |
| 266 | - "!vllm/distributed/kv_transfer/" |
| 267 | - tests/v1/distributed |
| 268 | - tests/v1/engine/test_engine_core_client.py |
| 269 | - tests/distributed/test_utils |
| 270 | - tests/distributed/test_engram_dp_shard.py |
| 271 | - vllm/config/engram.py |
| 272 | - vllm/models/deepseek_v41/ |
| 273 | commands: |
| 274 | # https://github.com/NVIDIA/nccl/issues/1838 |
| 275 | - export NCCL_CUMEM_HOST_ENABLE=0 |
| 276 | - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py |
| 277 | - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py |
| 278 | - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py |
| 279 | - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py |
| 280 | - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py |
| 281 | - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py |
| 282 | - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp |
| 283 | - pytest -v -s distributed/test_utils.py |
| 284 | - pytest -v -s distributed/test_engram_dp_shard.py |
| 285 | |
| 286 | - label: ":nvidia: (L4) Distributed Compile + Comm" |
| 287 | key: distributed-compile-comm-4-gpus |
| 288 | timeout_in_minutes: 70 |
| 289 | working_dir: "/vllm-workspace/tests" |
| 290 | device: l4 |
| 291 | num_devices: 4 |
| 292 | source_file_dependencies: |
| 293 | - vllm/distributed/ |
| 294 | - "!vllm/distributed/kv_transfer/" |
| 295 | - tests/distributed/test_pynccl |
| 296 | - tests/distributed/test_events |
| 297 | - tests/distributed/test_symm_mem_allreduce.py |
| 298 | - tests/distributed/test_multiproc_executor.py |
| 299 | commands: |
| 300 | # https://github.com/NVIDIA/nccl/issues/1838 |
| 301 | - export NCCL_CUMEM_HOST_ENABLE=0 |
| 302 | - pytest -v -s distributed/test_pynccl.py |
| 303 | - pytest -v -s distributed/test_events.py |
| 304 | - pytest -v -s distributed/test_symm_mem_allreduce.py |
| 305 | # test multi-node TP with multiproc executor (simulated on single node) |
| 306 | - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node |
| 307 | |
| 308 | - label: ":nvidia: (H100) Distributed DP + EP" |
| 309 | key: distributed-tests-8xh100 |
| 310 | timeout_in_minutes: 20 |
| 311 | device: h100 |
| 312 | num_devices: 8 |
| 313 | working_dir: "/vllm-workspace/tests" |
| 314 | source_file_dependencies: |
| 315 | - examples/features/torchrun/torchrun_dp_example_offline.py |
| 316 | - vllm/config/parallel.py |
| 317 | - vllm/distributed/ |
| 318 | - "!vllm/distributed/kv_transfer/" |
| 319 | - vllm/v1/engine/llm_engine.py |
| 320 | - vllm/v1/executor/uniproc_executor.py |
| 321 | - vllm/v1/worker/gpu_worker.py |
| 322 | - tests/distributed/test_mnnvl_alltoall.py |
| 323 | |
| 324 | commands: |
| 325 | # https://github.com/NVIDIA/nccl/issues/1838 |
| 326 | - export NCCL_CUMEM_HOST_ENABLE=0 |
| 327 | # test with torchrun tp=2 and dp=4 with ep |
| 328 | - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep |
| 329 | mirror: |
| 330 | amd: |
| 331 | label: ":amd: (MI355) Distributed DP + EP" |
| 332 | dind: false |
| 333 | device: mi355_8 |
| 334 | timeout_in_minutes: 50 |
| 335 | depends_on: |
| 336 | - image-build-amd |
| 337 | source_file_dependencies: |
| 338 | - examples/features/torchrun/torchrun_dp_example_offline.py |
| 339 | - vllm/config/parallel.py |
| 340 | - vllm/distributed/ |
| 341 | - "!vllm/distributed/kv_transfer/" |
| 342 | - vllm/v1/engine/llm_engine.py |
| 343 | - vllm/v1/executor/uniproc_executor.py |
| 344 | - vllm/v1/worker/gpu_worker.py |
| 345 | - vllm/platforms/rocm.py |
| 346 | |
| 347 | - label: ":nvidia: (A100) Distributed" |
| 348 | key: distributed-tests-4xa100 |
| 349 | device: a100 |
| 350 | optional: true |
| 351 | num_devices: 4 |
| 352 | source_file_dependencies: |
| 353 | - vllm/ |
| 354 | - "!vllm/distributed/kv_transfer/" |
| 355 | commands: |
| 356 | # NOTE: don't test llama model here, it seems hf implementation is buggy |
| 357 | # see https://github.com/vllm-project/vllm/pull/5689 for details |
| 358 | - pytest -v -s distributed/test_custom_all_reduce.py |
| 359 | - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py |
| 360 | - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' |
| 361 | - pytest -v -s -x lora/test_mixtral.py |
| 362 | |
| 363 | - label: ":nvidia: (H100) Distributed Features" |
| 364 | key: distributed-tests-2xh100-2xmi300 |
| 365 | timeout_in_minutes: 30 |
| 366 | device: h100 |
| 367 | optional: true |
| 368 | working_dir: "/vllm-workspace/" |
| 369 | num_devices: 2 |
| 370 | commands: |
| 371 | - pytest -v -s tests/distributed/test_context_parallel.py |
| 372 | - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py |
| 373 | - VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput |
| 374 | - pytest -v -s tests/v1/distributed/test_dbo.py |
| 375 | - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py |
| 376 | - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer_m2n.py |
| 377 | - pytest -v -s tests/distributed/test_packed_tensor.py |
| 378 | mirror: |
| 379 | amd: |
| 380 | label: ':amd: (MI355) Distributed Features' |
| 381 | dind: false |
| 382 | device: mi355_2 |
| 383 | timeout_in_minutes: 90 |
| 384 | working_dir: /vllm-workspace/ |
| 385 | depends_on: |
| 386 | - image-build-amd |
| 387 | source_file_dependencies: |
| 388 | - vllm/distributed/ |
| 389 | - vllm/v1/distributed/ |
| 390 | - vllm/model_executor/layers/fused_moe/ |
| 391 | - vllm/v1/attention/backends/ |
| 392 | - vllm/v1/attention/selector.py |
| 393 | - tests/v1/distributed/test_dbo.py |
| 394 | - tests/distributed/test_context_parallel.py |
| 395 | - examples/features/data_parallel/data_parallel_offline.py |
| 396 | - vllm/_aiter_ops.py |
| 397 | - vllm/platforms/rocm.py |
| 398 | - csrc/custom_quickreduce.cu |
| 399 | - csrc/ops.h |
| 400 | - csrc/torch_bindings.cpp |
| 401 | - vllm/model_executor/layers/ |
| 402 | - vllm/entrypoints/llm.py |
| 403 | - vllm/config/parallel.py |
| 404 | - vllm/v1/engine/ |
| 405 | - vllm/v1/executor/ |
| 406 | - vllm/v1/worker/ |
| 407 | - vllm/_custom_ops.py |
| 408 | - vllm/envs.py |
| 409 | - tests/distributed/test_rocm_aiter_custom_ar.py |
| 410 | - tests/distributed/test_rocm_quick_reduce.py |
| 411 | - tests/distributed/test_quick_all_reduce.py |
| 412 | - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py |
| 413 | - tests/utils.py |
| 414 | - examples/rl/rlhf_async_new_apis.py |
| 415 | - tests/distributed/test_weight_transfer.py |
| 416 | - tests/distributed/test_packed_tensor.py |
| 417 | commands: |
| 418 | - pytest -v -s tests/distributed/test_context_parallel.py |
| 419 | - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py |
| 420 | - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput |
| 421 | - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization |
| 422 | - pytest -v -s tests/v1/distributed/test_dbo.py |
| 423 | - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py |
| 424 | - pytest -v -s tests/distributed/test_packed_tensor.py |
| 425 | - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py |
| 426 | - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py |
| 427 | - pytest -v -s tests/distributed/test_rocm_quick_reduce.py |
| 428 | - pytest -v -s tests/distributed/test_quick_all_reduce.py |
| 429 | |
| 430 | - label: ":nvidia: (B200) Distributed" |
| 431 | key: distributed-tests-2xb200 |
| 432 | device: b200-k8s |
| 433 | optional: true |
| 434 | working_dir: "/vllm-workspace/" |
| 435 | num_devices: 2 |
| 436 | commands: |
| 437 | - pytest -v -s tests/distributed/test_custom_all_gather_reduce_scatter.py |
| 438 | - pytest -v -s tests/distributed/test_context_parallel.py |
| 439 | - pytest -v -s tests/distributed/test_nccl_symm_mem.py |
| 440 | - pytest -v -s tests/v1/distributed/test_dbo.py |
| 441 | - pytest -v -s tests/distributed/test_mnnvl_alltoall.py |
| 442 | mirror: |
| 443 | amd: |
| 444 | label: ":amd: (MI355) Distributed AgRs All2All" |
| 445 | dind: false |
| 446 | device: mi355_2 |
| 447 | num_devices: 2 |
| 448 | timeout_in_minutes: 30 |
| 449 | working_dir: "/vllm-workspace/" |
| 450 | depends_on: |
| 451 | - image-build-amd |
| 452 | source_file_dependencies: |
| 453 | - .buildkite/test_areas/distributed.yaml |
| 454 | - vllm/distributed/ |
| 455 | - vllm/config/parallel.py |
| 456 | - vllm/config/vllm.py |
| 457 | - vllm/forward_context.py |
| 458 | - vllm/platforms/rocm.py |
| 459 | - tests/distributed/test_mnnvl_alltoall.py |
| 460 | - tests/utils.py |
| 461 | commands: |
| 462 | - pytest -v -s tests/distributed/test_mnnvl_alltoall.py::test_args_dispatch_combine |
| 463 | |
| 464 | - label: ":nvidia: (B200) GEMM-RS/AR" |
| 465 | key: gemm-rs-ar-2xb200 |
| 466 | timeout_in_minutes: 20 |
| 467 | device: b200-k8s |
| 468 | working_dir: "/vllm-workspace/" |
| 469 | num_devices: 2 |
| 470 | source_file_dependencies: |
| 471 | - vllm/model_executor/kernels/linear/cute_dsl/gemm_rs_ar.py |
| 472 | - vllm/cute_utils/ |
| 473 | - tests/kernels/test_gemm_rs_ar.py |
| 474 | commands: |
| 475 | - pytest -v -s tests/kernels/test_gemm_rs_ar.py |
| 476 | mirror: |
| 477 | amd: |
| 478 | label: ":amd: (MI355) GEMM-RS-AR" |
| 479 | dind: false |
| 480 | device: mi355_4 |
| 481 | num_devices: 4 |
| 482 | timeout_in_minutes: 20 |
| 483 | working_dir: /vllm-workspace/ |
| 484 | depends_on: |
| 485 | - image-build-amd |
| 486 | source_file_dependencies: |
| 487 | - .buildkite/test_areas/distributed.yaml |
| 488 | - tests/kernels/test_gemm_rs_ar.py |
| 489 | - tests/kernels/test_rocm_gemm_rs_ar.py |
| 490 | - tests/conftest.py |
| 491 | - vllm/models/common/ops/sequence_parallel.py |
| 492 | - vllm/model_executor/layers/linear.py |
| 493 | - vllm/model_executor/layers/utils.py |
| 494 | - vllm/model_executor/layers/quantization/online/ |
| 495 | - vllm/model_executor/layers/quantization/utils/mxfp8_utils.py |
| 496 | - vllm/model_executor/kernels/linear/ |
| 497 | - vllm/distributed/ |
| 498 | - vllm/_aiter_ops.py |
| 499 | - vllm/_custom_ops.py |
| 500 | - csrc/rocm/skinny_gemms.cu |
| 501 | - vllm/platforms/rocm.py |
| 502 | commands: |
| 503 | - pytest -v -s tests/kernels/test_rocm_gemm_rs_ar.py |
| 504 | |
| 505 | - label: ":nvidia: (L4) Distributed 2-Node" |
| 506 | key: 2-node-test-4-gpus |
| 507 | timeout_in_minutes: 30 |
| 508 | working_dir: "/vllm-workspace/tests" |
| 509 | num_devices: 2 |
| 510 | num_nodes: 2 |
| 511 | no_plugin: true |
| 512 | optional: true # TODO: revert once infra issue solved |
| 513 | source_file_dependencies: |
| 514 | - vllm/distributed/ |
| 515 | - "!vllm/distributed/kv_transfer/" |
| 516 | - vllm/engine/ |
| 517 | - vllm/v1/executor/ |
| 518 | - vllm/model_executor/models/ |
| 519 | - tests/distributed/ |
| 520 | - examples/features/data_parallel/data_parallel_offline.py |
| 521 | commands: |
| 522 | - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 $IMAGE_TAG "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/features/data_parallel/data_parallel_offline.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/features/data_parallel/data_parallel_offline.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code" |
| 523 | |
| 524 | - label: ":nvidia: (L4) Pipeline + Context Parallelism" |
| 525 | key: pipeline-context-parallelism-4-gpus |
| 526 | timeout_in_minutes: 55 |
| 527 | working_dir: "/vllm-workspace/tests" |
| 528 | device: l4 |
| 529 | num_devices: 4 |
| 530 | source_file_dependencies: |
| 531 | - vllm/distributed/ |
| 532 | - "!vllm/distributed/kv_transfer/" |
| 533 | - vllm/engine/ |
| 534 | - vllm/v1/executor/ |
| 535 | - vllm/model_executor/models/ |
| 536 | - vllm/v1/worker/gpu/ |
| 537 | - vllm/v1/worker/gpu_worker.py |
| 538 | - tests/distributed/ |
| 539 | - tests/v1/distributed/test_pp_dp_v2.py |
| 540 | commands: |
| 541 | - pytest -v -s distributed/test_pp_cudagraph.py |
| 542 | - pytest -v -s distributed/test_pipeline_parallel.py |
| 543 | - pytest -v -s v1/distributed/test_pp_dp_v2.py |
| 544 | mirror: |
| 545 | amd: |
| 546 | label: ":amd: (MI250) Pipeline + Context Parallelism" |
| 547 | dind: false |
| 548 | device: mi250_4 |
| 549 | timeout_in_minutes: 90 |
| 550 | depends_on: |
| 551 | - image-build-amd |
| 552 | source_file_dependencies: |
| 553 | - vllm/distributed/ |
| 554 | - "!vllm/distributed/kv_transfer/" |
| 555 | - vllm/engine/ |
| 556 | - vllm/v1/executor/ |
| 557 | - vllm/model_executor/models/ |
| 558 | - vllm/model_executor/layers/ |
| 559 | - vllm/v1/attention/backends/ |
| 560 | - vllm/v1/attention/selector.py |
| 561 | - tests/distributed/ |
| 562 | - vllm/_aiter_ops.py |
| 563 | - vllm/platforms/rocm.py |
| 564 | |
| 565 | - label: ":nvidia: (L4) RayExecutorV2" |
| 566 | key: rayexecutorv2-4-gpus |
| 567 | timeout_in_minutes: 45 |
| 568 | working_dir: "/vllm-workspace/tests" |
| 569 | device: l4 |
| 570 | num_devices: 4 |
| 571 | source_file_dependencies: |
| 572 | - vllm/v1/executor/ray_executor_v2.py |
| 573 | - vllm/v1/executor/abstract.py |
| 574 | - vllm/v1/executor/multiproc_executor.py |
| 575 | - tests/distributed/test_ray_v2_executor.py |
| 576 | - tests/distributed/test_ray_v2_executor_e2e.py |
| 577 | - tests/distributed/test_pipeline_parallel.py |
| 578 | - tests/basic_correctness/models/test_basic_correctness.py |
| 579 | commands: |
| 580 | - export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 |
| 581 | - export NCCL_CUMEM_HOST_ENABLE=0 |
| 582 | - pytest -v -s distributed/test_ray_v2_executor.py |
| 583 | - pytest -v -s distributed/test_ray_v2_executor_e2e.py |
| 584 | - pytest -v -s distributed/test_pipeline_parallel.py -k "ray" |
| 585 | - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray" |