A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bug][Frontend] Improve ZMQ client robustness (#7443)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Aug 21, 2024 at 20:18 UTC
cde9183b40a88f4210a7e965a430ae860aba5f6d
6 files changed
+176
-28
tests/entrypoints/openai/rpc/__init__.py
tests/entrypoints/openai/rpc/test_zmq_client.py
new
+119
@@ -0,0 +1,119 @@
1
+import asyncio
2
+import tempfile
3
+import unittest
4
+import unittest.mock
5
+import uuid
6
+
7
+import pytest
8
+import pytest_asyncio
9
+
10
+from vllm.engine.async_llm_engine import AsyncLLMEngine
11
+from vllm.entrypoints.openai.rpc.client import (AsyncEngineRPCClient,
12
+ RPCClientClosedError)
13
+from vllm.entrypoints.openai.rpc.server import AsyncEngineRPCServer
14
+
15
+
16
+@pytest.fixture(scope="function")
17
+def tmp_socket():
18
+ with tempfile.TemporaryDirectory() as td:
19
+ yield f"ipc://{td}/{uuid.uuid4()}"
20
+
21
+
22
+@pytest_asyncio.fixture(scope="function")
23
+async def dummy_server(tmp_socket, monkeypatch):
24
+ dummy_engine = unittest.mock.AsyncMock()
25
+
26
+ def dummy_engine_builder(*args, **kwargs):
27
+ return dummy_engine
28
+
29
+ with monkeypatch.context() as m:
30
+ m.setattr(AsyncLLMEngine, "from_engine_args", dummy_engine_builder)
31
+ server = AsyncEngineRPCServer(None, None, rpc_path=tmp_socket)
32
+
33
+ loop = asyncio.get_running_loop()
34
+ server_task = loop.create_task(server.run_server_loop())
35
+
36
+ try:
37
+ yield server
38
+ finally:
39
+ server_task.cancel()
40
+ server.cleanup()
41
+
42
+
43
+@pytest_asyncio.fixture(scope="function")
44
+async def client(tmp_socket):
45
+ client = AsyncEngineRPCClient(rpc_path=tmp_socket)
46
+ # Sanity check: the server is connected
47
+ await client._wait_for_server_rpc()
48
+
49
+ try:
50
+ yield client
51
+ finally:
52
+ client.close()
53
+
54
+
55
+@pytest.mark.asyncio
56
+async def test_client_data_methods_use_timeouts(monkeypatch, dummy_server,
57
+ client: AsyncEngineRPCClient):
58
+ with monkeypatch.context() as m:
59
+ # Make the server _not_ reply with a model config
60
+ m.setattr(dummy_server, "get_config", lambda x: None)
61
+ m.setattr(client, "_data_timeout", 10)
62
+
63
+ # And ensure the task completes anyway
64
+ # (client.setup() invokes server.get_config())
65
+ client_task = asyncio.get_running_loop().create_task(client.setup())
66
+ with pytest.raises(TimeoutError, match="Server didn't reply within"):
67
+ await asyncio.wait_for(client_task, timeout=0.05)
68
+
69
+
70
+@pytest.mark.asyncio
71
+async def test_client_aborts_use_timeouts(monkeypatch, dummy_server,
72
+ client: AsyncEngineRPCClient):
73
+ with monkeypatch.context() as m:
74
+ # Hang all abort requests
75
+ m.setattr(dummy_server, "abort", lambda x: None)
76
+ m.setattr(client, "_data_timeout", 10)
77
+
78
+ # Ensure the client doesn't hang
79
+ client_task = asyncio.get_running_loop().create_task(
80
+ client.abort("test request id"))
81
+ with pytest.raises(TimeoutError, match="Server didn't reply within"):
82
+ await asyncio.wait_for(client_task, timeout=0.05)
83
+
84
+
85
+@pytest.mark.asyncio
86
+async def test_client_data_methods_reraise_exceptions(
87
+ monkeypatch, dummy_server, client: AsyncEngineRPCClient):
88
+ with monkeypatch.context() as m:
89
+ # Make the server raise some random exception
90
+ exception = RuntimeError("Client test exception")
91
+
92
+ def raiser():
93
+ raise exception
94
+
95
+ m.setattr(dummy_server.engine, "get_model_config", raiser)
96
+ m.setattr(client, "_data_timeout", 10)
97
+
98
+ client_task = asyncio.get_running_loop().create_task(client.setup())
99
+ # And ensure the task completes, raising the exception
100
+ with pytest.raises(RuntimeError, match=str(exception)):
101
+ await asyncio.wait_for(client_task, timeout=0.05)
102
+
103
+
104
+@pytest.mark.asyncio
105
+async def test_client_errors_after_closing(monkeypatch, dummy_server,
106
+ client: AsyncEngineRPCClient):
107
+
108
+ client.close()
109
+
110
+ # Healthchecks and generate requests will fail with explicit errors
111
+ with pytest.raises(RPCClientClosedError):
112
+ await client.check_health()
113
+ with pytest.raises(RPCClientClosedError):
114
+ async for _ in client.generate(None, None, None):
115
+ pass
116
+
117
+ # But no-ops like aborting will pass
118
+ await client.abort("test-request-id")
119
+ await client.do_log_stats()
vllm/entrypoints/openai/api_server.py
+3
-2
@@ -6,7 +6,7 @@ import os
6
import re
7
import tempfile
8
from argparse import Namespace
9
-from contextlib import asynccontextmanager
9
+from contextlib import asynccontextmanager, suppress
10
from http import HTTPStatus
11
from typing import AsyncIterator, Optional, Set
12
@@ -83,7 +83,8 @@ async def lifespan(app: FastAPI):
83
async def _force_log():
84
while True:
85
await asyncio.sleep(10)
86
- await async_engine_client.do_log_stats()
86
+ with suppress(Exception):
87
+ await async_engine_client.do_log_stats()
88
89
if not engine_args.disable_log_stats:
90
task = asyncio.create_task(_force_log())
vllm/entrypoints/openai/rpc/__init__.py
-4
@@ -10,10 +10,6 @@ from vllm.sampling_params import SamplingParams
10
# Success string used for RPC instructions.
11
VLLM_RPC_SUCCESS_STR = "SUCCESS"
12
13
-# Timeouts.
14
-VLLM_RPC_SERVER_START_TIMEOUT_MS = 1000
15
-VLLM_RPC_HEALTH_TIMEOUT_MS = 10000
16
-
13
# Minimum value of ZMQ.SOCKET_LIMIT to run mp.
14
VLLM_RPC_SOCKET_LIMIT_CUTOFF = 2000
15
vllm/entrypoints/openai/rpc/client.py
+48
-22
@@ -1,5 +1,5 @@
1
import asyncio
2
-from contextlib import contextmanager
2
+from contextlib import contextmanager, suppress
3
from typing import Any, AsyncGenerator, Mapping, Optional
4
from uuid import uuid4
5
@@ -11,13 +11,12 @@ from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11
ParallelConfig, SchedulerConfig)
12
# yapf: disable
13
from vllm.entrypoints.openai.rpc import (RPC_REQUEST_TYPE,
14
- VLLM_RPC_HEALTH_TIMEOUT_MS,
15
- VLLM_RPC_SERVER_START_TIMEOUT_MS,
14
VLLM_RPC_SOCKET_LIMIT_CUTOFF,
15
VLLM_RPC_SUCCESS_STR,
16
VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
17
RPCGenerateRequest, RPCUtilityRequest)
18
# yapf: enable
19
+from vllm.envs import VLLM_RPC_GET_DATA_TIMEOUT_MS
20
from vllm.inputs import PromptInputs
21
from vllm.logger import init_logger
22
from vllm.lora.request import LoRARequest
@@ -32,6 +31,17 @@ logger = init_logger(__name__)
31
INPROC_PROXY_PATH = f"inproc://{uuid4()}"
32
33
34
+class RPCClientClosedError(Exception):
35
+ """Exception class raised when the client is used post-close.
36
+
37
+ The client can be closed, which closes the ZMQ context. This normally
38
+ happens on server shutdown. In some cases, methods like abort and
39
+ do_log_stats will still be called and then try to open a socket, which
40
+ causes a ZMQError and creates a huge stack trace.
41
+ So, we throw this error such that we can suppress it.
42
+ """
43
+
44
+
45
class AsyncEngineRPCClient:
46
"""
47
RPCClient that connects to the RPCServer wrapping AsyncLLMEngine.
@@ -85,6 +95,8 @@ class AsyncEngineRPCClient:
95
96
def __init__(self, rpc_path: str):
97
self.context = zmq.asyncio.Context()
98
+ self._data_timeout = VLLM_RPC_GET_DATA_TIMEOUT_MS
99
+ self._errored = False
100
101
# Maximum number of sockets that can be opened (typically 65536).
102
# ZMQ_SOCKET_LIMIT (http://api.zeromq.org/4-2:zmq-ctx-get)
@@ -143,7 +155,6 @@ class AsyncEngineRPCClient:
155
156
# Wait until server is ready.
157
await self._wait_for_server_rpc()
146
- self._errored = False
158
159
# Get the configs.
160
self.model_config = await self._get_model_config_rpc()
@@ -170,6 +181,15 @@ class AsyncEngineRPCClient:
181
@contextmanager
182
def to_proxy_socket(self):
183
# Connect to the RPCServer via the proxy.
184
+
185
+ # Raise a sensible error if the client was already closed.
186
+ # This can happen if a server shutdown is triggered but some coroutines
187
+ # are still running requests.
188
+ # There should not be a race condition with this check because we don't
189
+ # yield to the event loop between here and opening the socket.
190
+ if self.context.closed:
191
+ raise RPCClientClosedError("The ZMQ client has already shut down")
192
+
193
# Note that we use DEALER to enable asynchronous communication
194
# to enable streaming.
195
socket = self.context.socket(zmq.constants.DEALER)
@@ -189,9 +209,18 @@ class AsyncEngineRPCClient:
209
# Ping RPCServer with a request.
210
await socket.send_multipart([cloudpickle.dumps(request)])
211
212
+ # Make sure the server responds
213
+ if await socket.poll(timeout=self._data_timeout) == 0:
214
+ raise TimeoutError("Server didn't reply within "
215
+ f"{self._data_timeout} ms")
216
+
217
# Await the data from the Server.
218
data = cloudpickle.loads(await socket.recv())
219
220
+ if isinstance(data, Exception):
221
+ # Re-raise exceptions returned by the server
222
+ raise data
223
+
224
if not isinstance(data, expected_type):
225
# LoRAConfig can be None.
226
if expected_type == LoRAConfig and data is None:
@@ -208,29 +237,28 @@ class AsyncEngineRPCClient:
237
self,
238
request: RPC_REQUEST_TYPE,
239
error_message: str,
211
- timeout: Optional[int] = None,
240
socket: Optional[zmq.asyncio.Socket] = None):
241
"""Send one-way RPC request to trigger an action."""
242
243
async def do_rpc_call(socket: zmq.asyncio.Socket,
216
- request: RPC_REQUEST_TYPE,
217
- timeout=None):
244
+ request: RPC_REQUEST_TYPE):
245
246
await socket.send_multipart([cloudpickle.dumps(request)])
247
221
- if timeout is not None and await socket.poll(timeout=timeout) == 0:
222
- raise TimeoutError(f"Server didn't reply within {timeout} ms")
248
+ if await socket.poll(timeout=self._data_timeout) == 0:
249
+ raise TimeoutError("Server didn't reply within "
250
+ f"{self._data_timeout} ms")
251
252
return cloudpickle.loads(await socket.recv())
253
254
# Make a new socket connection.
255
if socket is None:
256
with self.to_proxy_socket() as socket:
229
- response = await do_rpc_call(socket, request, timeout)
257
+ response = await do_rpc_call(socket, request)
258
259
# Use existing socket connection.
260
else:
233
- response = await do_rpc_call(socket, request, timeout)
261
+ response = await do_rpc_call(socket, request)
262
263
if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
264
if isinstance(response, Exception):
@@ -255,8 +283,7 @@ class AsyncEngineRPCClient:
283
284
await self._send_one_way_rpc_request(
285
request=RPCUtilityRequest.IS_SERVER_READY,
258
- error_message="Unable to start RPC Server",
259
- timeout=VLLM_RPC_SERVER_START_TIMEOUT_MS)
286
+ error_message="Unable to start RPC Server")
287
288
async def _get_model_config_rpc(self) -> ModelConfig:
289
"""Get the ModelConfig object from the RPC Server"""
@@ -308,17 +335,17 @@ class AsyncEngineRPCClient:
335
336
async def abort(self, request_id: str):
337
"""Send an ABORT_REQUEST signal to the RPC Server"""
311
-
312
- await self._send_one_way_rpc_request(
313
- request=RPCAbortRequest(request_id),
314
- error_message=f"RPCAbortRequest {request_id} failed")
338
+ with suppress(RPCClientClosedError):
339
+ await self._send_one_way_rpc_request(
340
+ request=RPCAbortRequest(request_id),
341
+ error_message=f"RPCAbortRequest {request_id} failed")
342
343
async def do_log_stats(self):
344
"""Send a DO_LOG_STATS signal to the RPC Server"""
318
-
319
- await self._send_one_way_rpc_request(
320
- request=RPCUtilityRequest.DO_LOG_STATS,
321
- error_message="RPCRequest DO_LOG_STATS failed.")
345
+ with suppress(RPCClientClosedError):
346
+ await self._send_one_way_rpc_request(
347
+ request=RPCUtilityRequest.DO_LOG_STATS,
348
+ error_message="RPCRequest DO_LOG_STATS failed.")
349
350
@property
351
def is_running(self) -> bool:
@@ -393,7 +420,6 @@ class AsyncEngineRPCClient:
420
await self._send_one_way_rpc_request(
421
request=RPCUtilityRequest.IS_SERVER_HEALTHY,
422
error_message="Got Unhealthy response from RPC Server",
396
- timeout=VLLM_RPC_HEALTH_TIMEOUT_MS,
423
socket=socket)
424
425
async def encode(self, *args,
vllm/envs.py
+6
@@ -56,6 +56,7 @@ if TYPE_CHECKING:
56
VERBOSE: bool = False
57
VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
58
VLLM_TEST_FORCE_FP8_MARLIN: bool = False
59
+ VLLM_RPC_GET_DATA_TIMEOUT_MS: int = 5000
60
VLLM_ALLOW_ENGINE_USE_RAY: bool = False
61
VLLM_PLUGINS: Optional[List[str]] = None
62
VLLM_TORCH_PROFILER_DIR: Optional[str] = None
@@ -374,6 +375,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
375
(os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
376
("1", "true")),
377
378
+ # Time in ms for the zmq client to wait for a response from the backend
379
+ # server for simple data operations
380
+ "VLLM_RPC_GET_DATA_TIMEOUT_MS":
381
+ lambda: int(os.getenv("VLLM_RPC_GET_DATA_TIMEOUT_MS", "5000")),
382
+
383
# If set, allow running the engine as a separate ray actor,
384
# which is a deprecated feature soon to be removed.
385
# See https://github.com/vllm-project/vllm/issues/7045