A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 00:00 UTC
[ROCm][CI] Stabilize ROCm shutdown and distributed compile CI (#41573)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
May 9, 2026 at 22:47 UTC
fb1ac806c55a6dc96fe92261b80c8550e9c39d2f
2 files changed
+12
-9
.buildkite/test-amd.yaml
+2
-2
@@ -1100,13 +1100,13 @@ steps:
1100
- vllm/compilation/
1101
- vllm/model_executor/layers
1102
- tests/compile/passes/distributed/
1103
+ - tests/compile/fusions_e2e/
1104
- vllm/_aiter_ops.py
1105
- vllm/platforms/rocm.py
1106
commands:
1107
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
1108
- VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py
1108
- - pytest -v -s tests/compile/passes/distributed/test_sequence_parallelism.py
1109
- - pytest -v -s tests/compile/passes/distributed/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions
1109
+ - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions
1110
1111
#----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------#
1112
tests/entrypoints/openai/completion/test_shutdown.py
+10
-7
@@ -299,19 +299,22 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
299
start_time = time.time()
300
proc.send_signal(signal.SIGTERM)
301
302
- # abort timeout (0) should exit promptly
303
- for _ in range(20):
304
- if proc.poll() is not None:
305
- break
306
- time.sleep(0.1)
302
+ # abort timeout (0) should stop the server promptly. On ROCm, process
303
+ # exit can spend extra time in HIP/RCCL/native extension teardown after
304
+ # the server and engine have already shut down.
305
+ max_exit_time = 4.0 if _IS_ROCM else 2.1
306
308
- if proc.poll() is None:
307
+ try:
308
+ proc.wait(timeout=max_exit_time)
309
+ except subprocess.TimeoutExpired:
310
proc.kill()
311
proc.wait(timeout=5)
312
pytest.fail("Process did not exit after SIGTERM with abort timeout")
313
314
exit_time = time.time() - start_time
314
- assert exit_time < 2.1, f"Default shutdown took too long: {exit_time:.1f}s"
315
+ assert exit_time < max_exit_time, (
316
+ f"Default shutdown took too long: {exit_time:.1f}s"
317
+ )
318
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
319
320
await _assert_children_cleaned_up(child_pids)