A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 05:00 UTC
[ROCm][CI] Remove `TORCH_NCCL_BLOCKING_WAIT=1` After Bugfix In ROCm 7.2 (#41840)
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Micah Williamson committed
May 6, 2026 at 18:37 UTC
7a576e2c724e135236c0dc005dce114c7e4d911e
1 file changed
-16
.buildkite/test-amd.yaml
-16
@@ -230,7 +230,6 @@ steps:
230
- tests/entrypoints/llm/test_collective_rpc.py
231
- vllm/platforms/rocm.py
232
commands:
233
- - export TORCH_NCCL_BLOCKING_WAIT=1
233
- pytest -v -s entrypoints/llm/test_collective_rpc.py
234
- pytest -v -s ./compile/fullgraph/test_basic_correctness.py
235
- pytest -v -s ./compile/test_wrapper.py
@@ -272,7 +271,6 @@ steps:
271
- tests/v1/worker/test_worker_memory_snapshot.py
272
- vllm/platforms/rocm.py
273
commands:
275
- - export TORCH_NCCL_BLOCKING_WAIT=1
274
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
275
- VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
276
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
@@ -590,7 +588,6 @@ steps:
588
- vllm/platforms/rocm.py
589
commands:
590
- pip freeze | grep -E 'torch'
593
- - export TORCH_NCCL_BLOCKING_WAIT=1
591
- pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
592
593
- label: Multi-Modal Models (Extended Generation 2) # TBD
@@ -865,7 +862,6 @@ steps:
862
- tests/entrypoints/openai/test_multi_api_servers.py
863
- vllm/platforms/rocm.py
864
commands:
868
- - export TORCH_NCCL_BLOCKING_WAIT=1
865
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
866
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
867
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
@@ -1175,7 +1171,6 @@ steps:
1171
- vllm/_aiter_ops.py
1172
- vllm/platforms/rocm.py
1173
commands:
1178
- - export TORCH_NCCL_BLOCKING_WAIT=1
1174
- pytest -v -s tests/distributed/test_context_parallel.py
1175
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
1176
@@ -1189,7 +1184,6 @@ steps:
1184
source_file_dependencies:
1185
- vllm/
1186
commands:
1192
- - export TORCH_NCCL_BLOCKING_WAIT=1
1187
- pytest -v -s distributed/test_custom_all_reduce.py
1188
- torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
1189
- TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
@@ -1209,7 +1203,6 @@ steps:
1203
- tests/examples/features/data_parallel/data_parallel_offline.py
1204
- vllm/platforms/rocm.py
1205
commands:
1212
- - export TORCH_NCCL_BLOCKING_WAIT=1
1206
- torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
1207
- PP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
1208
- TP_SIZE=4 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
@@ -1255,7 +1248,6 @@ steps:
1248
- vllm/platforms/rocm.py
1249
commands:
1250
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
1258
- - export TORCH_NCCL_BLOCKING_WAIT=1
1251
- pytest -v -s distributed/test_ray_v2_executor.py
1252
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
1253
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
@@ -1277,7 +1269,6 @@ steps:
1269
- vllm/v1/worker/gpu_worker.py
1270
- vllm/platforms/rocm.py
1271
commands:
1280
- - export TORCH_NCCL_BLOCKING_WAIT=1
1272
- torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
1273
1274
#-------------------------------------------------------- mi300 · entrypoints --------------------------------------------------------#
@@ -2284,7 +2275,6 @@ steps:
2275
- tests/entrypoints/openai/test_multi_api_servers.py
2276
- vllm/platforms/rocm.py
2277
commands:
2287
- - export TORCH_NCCL_BLOCKING_WAIT=1
2278
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
2279
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
2280
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
@@ -2304,7 +2294,6 @@ steps:
2294
- vllm/_aiter_ops.py
2295
- vllm/platforms/rocm.py
2296
commands:
2307
- - export TORCH_NCCL_BLOCKING_WAIT=1
2297
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
2298
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
2299
- pytest -v -s tests/v1/distributed/test_dbo.py
@@ -2367,7 +2356,6 @@ steps:
2356
- tests/distributed/test_utils
2357
- vllm/platforms/rocm.py
2358
commands:
2370
- - export TORCH_NCCL_BLOCKING_WAIT=1
2359
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
2360
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
2361
- TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
@@ -2497,7 +2485,6 @@ steps:
2485
- tests/entrypoints/llm/test_collective_rpc.py
2486
- vllm/platforms/rocm.py
2487
commands:
2500
- - export TORCH_NCCL_BLOCKING_WAIT=1
2488
- pytest -v -s entrypoints/llm/test_collective_rpc.py
2489
- pytest -v -s ./compile/fullgraph/test_basic_correctness.py
2490
- pytest -v -s ./compile/test_wrapper.py
@@ -2522,7 +2509,6 @@ steps:
2509
- tests/v1/worker/test_worker_memory_snapshot.py
2510
- vllm/platforms/rocm.py
2511
commands:
2525
- - export TORCH_NCCL_BLOCKING_WAIT=1
2512
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
2513
- VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
2514
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
@@ -2543,7 +2529,6 @@ steps:
2529
- tests/distributed/test_multiproc_executor.py
2530
- vllm/platforms/rocm.py
2531
commands:
2546
- - export TORCH_NCCL_BLOCKING_WAIT=1
2532
- pytest -v -s compile/fullgraph/test_basic_correctness.py
2533
- pytest -v -s distributed/test_pynccl.py
2534
- pytest -v -s distributed/test_events.py
@@ -2723,7 +2708,6 @@ steps:
2708
- vllm/_aiter_ops.py
2709
- vllm/platforms/rocm.py
2710
commands:
2726
- - export TORCH_NCCL_BLOCKING_WAIT=1
2711
- pytest -v -s tests/distributed/test_context_parallel.py
2712
- pytest -v -s tests/v1/distributed/test_dbo.py
2713