[CI] Deflake shutdown wait-timeout test by synchronizing on request admission (#58125)
Signed-off-by: jackLei0901 <42642542+jackLei0901@users.noreply.github.com>
jackLei committed
Sep 30, 2026 at 17:10 UTC
21c08ac4589493291afa9c1471d08fb8a07a901a
1 file changed
+44
-4
tests/entrypoints/launchers/test_shutdown.py
+44
-4
@@ -14,6 +14,7 @@ import httpx
14
import openai
15
import psutil
16
import pytest
17
+from prometheus_client.parser import text_string_to_metric_families
18
19
from tests.utils import RemoteOpenAIServer
20
from vllm.utils.network_utils import get_open_port
@@ -127,6 +128,40 @@ async def _concurrent_request_loop(
128
t.cancel()
129
130
131
+def _num_running_requests(metrics_text: str) -> float:
132
+ """Sum ``vllm:num_requests_running`` across the server's label sets."""
133
+ total = 0.0
134
+ for family in text_string_to_metric_families(metrics_text):
135
+ if family.name == "vllm:num_requests_running":
136
+ for sample in family.samples:
137
+ if sample.name == "vllm:num_requests_running":
138
+ total += sample.value
139
+ return total
140
+
141
+
142
+async def _wait_for_running_request(
143
+ metrics_url: str,
144
+ timeout: float = _INFLIGHT_REQUEST_START_TIMEOUT,
145
+) -> float:
146
+ """Wait until the server reports a running request, returning the delay.
147
+
148
+ ``ShutdownState.inflight_requests`` is incremented before the HTTP call is
149
+ awaited, so it only shows that a client coroutine started. Draining can
150
+ only complete work the engine has already admitted, so synchronize on the
151
+ server's own scheduler gauge instead.
152
+ """
153
+ start = time.monotonic()
154
+ deadline = start + timeout
155
+ async with httpx.AsyncClient() as metrics_client:
156
+ while time.monotonic() < deadline:
157
+ response = await metrics_client.get(metrics_url)
158
+ response.raise_for_status()
159
+ if _num_running_requests(response.text) > 0:
160
+ return time.monotonic() - start
161
+ await asyncio.sleep(_INFLIGHT_REQUEST_POLL_INTERVAL)
162
+ raise AssertionError(f"No request reached the running state within {timeout}s")
163
+
164
+
165
@pytest.mark.asyncio
166
async def test_shutdown_on_engine_failure(tmp_path: Path):
167
"""Verify that API returns connection error when server process is killed.
@@ -240,11 +275,15 @@ async def test_wait_timeout_completes_requests():
275
_concurrent_request_loop(client, state, sigterm_sent, concurrency=10)
276
)
277
243
- await asyncio.sleep(0.5)
244
- proc.send_signal(signal.SIGTERM)
245
- sigterm_sent.set()
246
-
278
try:
279
+ # Drain can only complete requests the engine already admitted, so
280
+ # wait for the server to report one instead of guessing with a sleep.
281
+ admission_delay = await _wait_for_running_request(
282
+ remote_server.url_for("metrics")
283
+ )
284
+ proc.send_signal(signal.SIGTERM)
285
+ sigterm_sent.set()
286
+
287
await asyncio.wait_for(request_task, timeout=_SHUTDOWN_DETECTION_TIMEOUT)
288
except asyncio.TimeoutError:
289
pass
@@ -257,6 +296,7 @@ async def test_wait_timeout_completes_requests():
296
# wait timeout should complete in-flight requests
297
assert state.requests_after_sigterm > 0, (
298
f"Wait timeout should complete in-flight requests. "
299
+ f"admission_delay: {admission_delay:.3f}s, "
300
f"503: {state.got_503}, 500: {state.got_500}, "
301
f"conn_errors: {state.connection_errors}, errors: {state.errors}"
302
)