[CI][Bugfix] Fix `test_shutdown_on_engine_failure` startup deadlock (#51050)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Aug 4, 2026 at 18:29 UTC
77b519912bdba6b5a587df00c662cee663991cac
1 file changed
+39
-39
tests/entrypoints/openai/completion/test_shutdown.py
+39
-39
@@ -8,6 +8,7 @@ import subprocess
8
import sys
9
import time
10
from dataclasses import dataclass, field
11
+from pathlib import Path
12
13
import httpx
14
import openai
@@ -15,13 +16,11 @@ import psutil
16
import pytest
17
18
from tests.utils import RemoteOpenAIServer
18
-from vllm.platforms import current_platform
19
from vllm.utils.network_utils import get_open_port
20
21
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
22
23
# GPU initialization might take take longer
24
-_IS_ROCM = current_platform.is_rocm()
24
_SERVER_STARTUP_TIMEOUT = 120
25
_PROCESS_EXIT_TIMEOUT = 15
26
_SHUTDOWN_DETECTION_TIMEOUT = 10
@@ -129,7 +128,7 @@ async def _concurrent_request_loop(
128
129
130
@pytest.mark.asyncio
132
-async def test_shutdown_on_engine_failure():
131
+async def test_shutdown_on_engine_failure(tmp_path: Path):
132
"""Verify that API returns connection error when server process is killed.
133
134
Starts a vLLM server, kills it to simulate a crash, then verifies that
@@ -138,33 +137,35 @@ async def test_shutdown_on_engine_failure():
137
138
port = get_open_port()
139
141
- proc = subprocess.Popen(
142
- [
143
- # dtype, max-len etc set so that this can run in CI
144
- sys.executable,
145
- "-m",
146
- "vllm.entrypoints.openai.api_server",
147
- "--model",
148
- MODEL_NAME,
149
- "--dtype",
150
- "bfloat16",
151
- "--max-model-len",
152
- "128",
153
- "--enforce-eager",
154
- "--port",
155
- str(port),
156
- "--gpu-memory-utilization",
157
- "0.05",
158
- "--max-num-seqs",
159
- "2",
160
- ],
161
- # ROCm: Disable stdout/stderr pipe capture. Subprocess hangs when
162
- # stdout/stderr pipes are enabled during ROCm GPU initialization.
163
- stdout=None if _IS_ROCM else subprocess.PIPE,
164
- stderr=None if _IS_ROCM else subprocess.PIPE,
165
- text=None if _IS_ROCM else True,
166
- preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN),
167
- )
140
+ # Redirect server output to a file rather than a pipe: nothing drains the
141
+ # pipes while we poll for startup, so a server that writes more than the
142
+ # pipe buffer holds would block on write and never become ready.
143
+ server_log = tmp_path / "server.log"
144
+ with server_log.open("wb") as log_file:
145
+ proc = subprocess.Popen(
146
+ [
147
+ # dtype, max-len etc set so that this can run in CI
148
+ sys.executable,
149
+ "-m",
150
+ "vllm.entrypoints.openai.api_server",
151
+ "--model",
152
+ MODEL_NAME,
153
+ "--dtype",
154
+ "bfloat16",
155
+ "--max-model-len",
156
+ "128",
157
+ "--enforce-eager",
158
+ "--port",
159
+ str(port),
160
+ "--gpu-memory-utilization",
161
+ "0.05",
162
+ "--max-num-seqs",
163
+ "2",
164
+ ],
165
+ stdout=log_file,
166
+ stderr=subprocess.STDOUT,
167
+ preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN),
168
+ )
169
170
# Wait for server startup
171
start_time = time.time()
@@ -185,18 +186,17 @@ async def test_shutdown_on_engine_failure():
186
except Exception:
187
time.sleep(0.5)
188
if proc.poll() is not None:
188
- if _IS_ROCM:
189
- pytest.fail(f"Server died during startup: {proc.returncode}")
190
- else:
191
- stdout, stderr = proc.communicate(timeout=1)
192
- pytest.fail(
193
- f"Server died during startup. "
194
- f"stdout: {stdout}, stderr: {stderr}"
195
- )
189
+ pytest.fail(
190
+ f"Server died during startup: {proc.returncode}\n"
191
+ f"{server_log.read_text(errors='replace')}"
192
+ )
193
else:
194
proc.terminate()
195
proc.wait(timeout=_PROCESS_EXIT_TIMEOUT)
199
- pytest.fail(f"Server failed to start in {_SERVER_STARTUP_TIMEOUT} seconds")
196
+ pytest.fail(
197
+ f"Server failed to start in {_SERVER_STARTUP_TIMEOUT} seconds\n"
198
+ f"{server_log.read_text(errors='replace')}"
199
+ )
200
201
# Kill server to simulate crash
202
proc.terminate()