[CI] Harden RemoteVLLMServer GPU cleanup checks (#52282)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Andreas Karatzas committed
Aug 19, 2026 at 14:17 UTC
cb58bb9c1e38cd366910858873946c95be6328a9
3 files changed
+59
-9
tests/entrypoints/launchers/test_shutdown.py
+11
-7
@@ -272,7 +272,9 @@ async def test_wait_timeout_completes_requests():
272
273
@pytest.mark.asyncio
274
@pytest.mark.parametrize("wait_for_engine_idle", [0.0, 2.0])
275
-async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
275
+async def test_abort_timeout_exits_within_cleanup_grace(
276
+ wait_for_engine_idle: float,
277
+):
278
server_args = [
279
"--dtype",
280
"bfloat16",
@@ -303,19 +305,21 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
305
# Wait for engine to become idle
306
await asyncio.sleep(wait_for_engine_idle)
307
306
- start_time = time.time()
308
proc.send_signal(signal.SIGTERM)
309
309
- # abort timeout (0) should stop the server promptly.
310
+ # A zero request timeout aborts requests immediately, but process
311
+ # teardown may still use the platform-specific resource cleanup grace.
312
+ termination_timeout = remote_server._get_process_termination_timeout()
313
try:
311
- proc.wait(timeout=4.0)
314
+ proc.wait(timeout=termination_timeout)
315
except subprocess.TimeoutExpired:
316
proc.kill()
317
proc.wait(timeout=5)
315
- pytest.fail("Process did not exit after SIGTERM with abort timeout")
318
+ pytest.fail(
319
+ "Process did not exit within the resource cleanup grace "
320
+ f"({termination_timeout}s)"
321
+ )
322
317
- exit_time = time.time() - start_time
318
- assert exit_time < 4.1, f"Default shutdown took too long: {exit_time:.1f}s"
323
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
324
325
await _assert_children_cleaned_up(child_pids)
tests/entrypoints/unit_tests/test_remote_vllm_server.py
new
+33
@@ -0,0 +1,33 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from unittest.mock import Mock
5
+
6
+import pytest
7
+
8
+import tests.utils as test_utils
9
+from tests.utils import RemoteOpenAIServer
10
+
11
+
12
+def test_openai_server_shutdown_wait_covers_engine_cleanup(
13
+ monkeypatch: pytest.MonkeyPatch,
14
+):
15
+ server = object.__new__(RemoteOpenAIServer)
16
+ server._request_shutdown_timeout = 0.0
17
+ server.proc = Mock(pid=1234)
18
+ engine_timeout_args = []
19
+
20
+ def get_engine_timeout(request_timeout, process_timeout):
21
+ engine_timeout_args.append((request_timeout, process_timeout))
22
+ return 60.0
23
+
24
+ monkeypatch.setattr(
25
+ test_utils, "get_engine_process_shutdown_timeout", get_engine_timeout
26
+ )
27
+ monkeypatch.setattr(test_utils.os, "getpgid", lambda pid: pid)
28
+ monkeypatch.setattr(server, "_kill_process_group_survivors", Mock())
29
+
30
+ server._terminate_process_tree()
31
+
32
+ assert engine_timeout_args == [(0.0, 0.0)]
33
+ server.proc.wait.assert_called_once_with(timeout=75.0)
tests/utils.py
+15
-2
@@ -63,6 +63,7 @@ from vllm.utils.network_utils import get_open_port
63
from vllm.utils.torch_utils import (
64
set_random_seed, # noqa: F401 - re-exported for use in test files
65
)
66
+from vllm.v1.engine.utils import get_engine_process_shutdown_timeout
67
68
logger = init_logger(__name__)
69
@@ -237,6 +238,9 @@ class RemoteVLLMServer:
238
model_loader = get_model_loader(load_config)
239
model_loader.download_model(model_config)
240
241
+ def _get_process_termination_timeout(self) -> float:
242
+ return 15.0
243
+
244
def __init__(
245
self,
246
model: str,
@@ -287,6 +291,7 @@ class RemoteVLLMServer:
291
self.show_hidden_metrics = (
292
getattr(args, "show_hidden_metrics_for_version", None) is not None
293
)
294
+ self._request_shutdown_timeout = float(args.shutdown_timeout)
295
296
with _temporarily_sanitized_pythonpath_env():
297
self._pre_download_model(model, args)
@@ -416,7 +421,7 @@ class RemoteVLLMServer:
421
print(f"[RemoteOpenAIServer] Sent SIGTERM to process {pid}")
422
423
try:
419
- self.proc.wait(timeout=15)
424
+ self.proc.wait(timeout=self._get_process_termination_timeout())
425
print(f"[RemoteOpenAIServer] Server {pid} terminated gracefully")
426
except subprocess.TimeoutExpired:
427
# Phase 2: SIGKILL the entire process group
@@ -758,6 +763,14 @@ class RemoteVLLMServer:
763
class RemoteOpenAIServer(RemoteVLLMServer):
764
"""Launches ``vllm serve`` for testing OpenAI-compatible endpoints."""
765
766
+ def _get_process_termination_timeout(self) -> float:
767
+ engine_timeout = get_engine_process_shutdown_timeout(
768
+ self._request_shutdown_timeout,
769
+ self._request_shutdown_timeout,
770
+ )
771
+ assert engine_timeout is not None
772
+ return engine_timeout + super()._get_process_termination_timeout()
773
+
774
def _create_cli_subcommand(self):
775
return ServeSubcommand()
776
@@ -889,7 +902,7 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer):
902
self.proc.terminate()
903
print(f"[RemoteOpenAIServerCustom] Sent SIGTERM to process {pid}")
904
892
- self.proc.join(15)
905
+ self.proc.join(self._get_process_termination_timeout())
906
if self.proc.is_alive():
907
print(
908
f"[RemoteOpenAIServerCustom] Server {pid} did not respond "