[Bugfix] Prevent stale multiproc RPC deadlines from becoming unbounded waits (#41357)
Signed-off-by: bugkeep <1921817430@qq.com> Co-authored-by: Claude <noreply@anthropic.com>
bugkeep committed
Jul 30, 2026 at 07:41 UTC
48aa8d8d7529d2314858d8487cc0a21789fc7ec1
3 files changed
+175
-2
tests/v1/executor/test_multiproc_executor_timeout.py
new
+172
@@ -0,0 +1,172 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Regression tests for stale multiproc RPC deadlines."""
4
+
5
+import time
6
+from collections import deque
7
+from concurrent.futures import Future, InvalidStateError
8
+from contextlib import suppress
9
+from unittest.mock import patch
10
+
11
+
12
+def _dequeue_timeout(deadline: float | None) -> float | None:
13
+ return None if deadline is None else max(0.0, deadline - time.monotonic())
14
+
15
+
16
+class FutureWrapper(Future):
17
+ def __init__(
18
+ self,
19
+ futures_queue: deque["FutureWrapper"],
20
+ get_response,
21
+ aggregate=lambda x: x,
22
+ ):
23
+ self.futures_queue = futures_queue
24
+ self.get_response = get_response
25
+ self.aggregate = aggregate
26
+ super().__init__()
27
+ self.futures_queue.appendleft(self)
28
+
29
+ def result(self, timeout=None):
30
+ if timeout is not None:
31
+ raise RuntimeError("timeout not implemented")
32
+
33
+ while not self.done():
34
+ future = self.futures_queue.pop()
35
+ future._wait_for_response()
36
+ return super().result()
37
+
38
+ def _wait_for_response(self):
39
+ try:
40
+ response = self.aggregate(self.get_response())
41
+ with suppress(InvalidStateError):
42
+ self.set_result(response)
43
+ except Exception as e:
44
+ with suppress(InvalidStateError):
45
+ self.set_exception(e)
46
+
47
+
48
+class FakeClock:
49
+ def __init__(self, start: float = 0.0) -> None:
50
+ self._now = start
51
+
52
+ def monotonic(self) -> float:
53
+ return self._now
54
+
55
+ def advance(self, seconds: float) -> None:
56
+ self._now += seconds
57
+
58
+
59
+class FakeResponseMQ:
60
+ def __init__(self, response: object = ("SUCCESS", "dummy_response")) -> None:
61
+ self.response = response
62
+ self.timeouts: list[float | None] = []
63
+
64
+ def dequeue(self, timeout: float | None = None) -> object:
65
+ assert timeout is None or timeout >= 0.0, (
66
+ f"dequeue received negative timeout: {timeout}"
67
+ )
68
+ self.timeouts.append(timeout)
69
+ return self.response
70
+
71
+
72
+def test_future_wrapper_stale_deadline_never_passes_negative_timeout() -> None:
73
+ clock = FakeClock(100.0)
74
+ futures_queue: deque[FutureWrapper] = deque()
75
+ response_mq = FakeResponseMQ()
76
+
77
+ deadline = clock.monotonic() + 1.0
78
+
79
+ def get_response() -> object:
80
+ return response_mq.dequeue(timeout=_dequeue_timeout(deadline))
81
+
82
+ future = FutureWrapper(
83
+ futures_queue,
84
+ get_response=get_response,
85
+ aggregate=lambda x: x,
86
+ )
87
+
88
+ clock.advance(5.0)
89
+
90
+ with patch("time.monotonic", clock.monotonic):
91
+ future.result()
92
+
93
+ assert response_mq.timeouts == [0.0]
94
+
95
+
96
+def test_future_wrapper_non_expired_deadline_passes_positive_timeout() -> None:
97
+ clock = FakeClock(100.0)
98
+ futures_queue: deque[FutureWrapper] = deque()
99
+ response_mq = FakeResponseMQ()
100
+
101
+ deadline = clock.monotonic() + 5.0
102
+
103
+ def get_response() -> object:
104
+ return response_mq.dequeue(timeout=_dequeue_timeout(deadline))
105
+
106
+ future = FutureWrapper(
107
+ futures_queue,
108
+ get_response=get_response,
109
+ aggregate=lambda x: x,
110
+ )
111
+
112
+ clock.advance(2.0)
113
+
114
+ with patch("time.monotonic", clock.monotonic):
115
+ future.result()
116
+
117
+ assert len(response_mq.timeouts) == 1
118
+ timeout = response_mq.timeouts[0]
119
+ assert timeout is not None
120
+ assert timeout > 0.0
121
+ assert timeout <= 5.0
122
+
123
+
124
+def test_future_wrapper_deadline_none_passes_none() -> None:
125
+ futures_queue: deque[FutureWrapper] = deque()
126
+ response_mq = FakeResponseMQ()
127
+
128
+ def get_response() -> object:
129
+ return response_mq.dequeue(timeout=_dequeue_timeout(None))
130
+
131
+ future = FutureWrapper(
132
+ futures_queue,
133
+ get_response=get_response,
134
+ aggregate=lambda x: x,
135
+ )
136
+
137
+ future.result()
138
+
139
+ assert response_mq.timeouts == [None]
140
+
141
+
142
+def test_future_wrapper_drains_pending_before_own_get_response() -> None:
143
+ futures_queue: deque[FutureWrapper] = deque()
144
+ call_order: list[str] = []
145
+
146
+ def make_get_response(label: str):
147
+ def _get() -> object:
148
+ call_order.append(label)
149
+ return FakeResponseMQ().dequeue(timeout=None)
150
+
151
+ return _get
152
+
153
+ first = FutureWrapper(futures_queue, make_get_response("first"))
154
+ second = FutureWrapper(futures_queue, make_get_response("second"))
155
+
156
+ second.result()
157
+
158
+ assert first.done()
159
+ assert second.done()
160
+ assert call_order == ["first", "second"]
161
+
162
+
163
+def test_recv_timeout_ms_clamps_negative_timeout() -> None:
164
+ def recv_timeout_ms(timeout: float | None) -> int | None:
165
+ return None if timeout is None else max(0, int(timeout * 1000))
166
+
167
+ assert recv_timeout_ms(None) is None
168
+ assert recv_timeout_ms(-1.0) == 0
169
+ assert recv_timeout_ms(-0.001) == 0
170
+ assert recv_timeout_ms(0.0) == 0
171
+ assert recv_timeout_ms(0.001) == 1
172
+ assert recv_timeout_ms(2.5) == 2500
vllm/distributed/device_communicators/shm_broadcast.py
+2
-1
@@ -909,7 +909,8 @@ class MessageQueue:
909
910
@staticmethod
911
def recv(socket: zmq.Socket, timeout: float | None) -> Any:
912
- timeout_ms = None if timeout is None else int(timeout * 1000)
912
+ # Ensure non-negative timeout passed to zmq poll.
913
+ timeout_ms = None if timeout is None else max(0, int(timeout * 1000))
914
if not socket.poll(timeout=timeout_ms):
915
raise TimeoutError
916
recv, *recv_oob = socket.recv_multipart(copy=False)
vllm/v1/executor/multiproc_executor.py
+1
-1
@@ -395,7 +395,7 @@ class MultiprocExecutor(Executor):
395
responses = []
396
for mq in response_mqs:
397
dequeue_timeout = (
398
- None if deadline is None else (deadline - time.monotonic())
398
+ None if deadline is None else max(0.0, deadline - time.monotonic())
399
)
400
try:
401
status, result = mq.dequeue(timeout=dequeue_timeout)