A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 00:00 UTC
[ROCm][CI] Set TORCH_NCCL_BLOCKING_WAIT Distributed Tests On ROCm (#31259)
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Micah Williamson committed
Dec 24, 2025 at 01:19 UTC
6559d96796da12709389488a36ff706ee1db43c6
1 file changed
+9
-2
.buildkite/test-amd.yaml
+9
-2
@@ -222,6 +222,9 @@ steps:
222
- tests/v1/engine/test_engine_core_client.py
223
- tests/distributed/test_symm_mem_allreduce.py
224
commands:
225
+ # Work around HIP bug tracked here: https://github.com/ROCm/hip/issues/3876
226
+ # TODO: Remove when the bug is fixed in a future ROCm release
227
+ - export TORCH_NCCL_BLOCKING_WAIT=1
228
# test with torchrun tp=2 and external_dp=2
229
- torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
230
# test with torchrun tp=2 and pp=2
@@ -270,9 +273,10 @@ steps:
273
- vllm/v1/executor/uniproc_executor.py
274
- vllm/v1/worker/gpu_worker.py
275
commands:
273
- # https://github.com/NVIDIA/nccl/issues/1838
274
- #- export NCCL_CUMEM_HOST_ENABLE=0
276
# test with torchrun tp=2 and dp=4 with ep
277
+ # Work around HIP bug tracked here: https://github.com/ROCm/hip/issues/3876
278
+ # TODO: Remove when the bug is fixed in a future ROCm release
279
+ - export TORCH_NCCL_BLOCKING_WAIT=1
280
- torchrun --nproc-per-node=8 ../examples/offline_inference/torchrun_dp_example.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
281
282
- label: EPLB Algorithm Test # 5min
@@ -1294,6 +1298,9 @@ steps:
1298
- tests/v1/shutdown
1299
- tests/v1/worker/test_worker_memory_snapshot.py
1300
commands:
1301
+ # Work around HIP bug tracked here: https://github.com/ROCm/hip/issues/3876
1302
+ # TODO: Remove when the bug is fixed in a future ROCm release
1303
+ - export TORCH_NCCL_BLOCKING_WAIT=1
1304
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
1305
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
1306
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py