@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cde9183b40
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bug][Frontend] Improve ZMQ client robustness (#7443)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Aug 21, 2024 at 20:18 UTC cde9183b40a88f4210a7e965a430ae860aba5f6d
6 files changed +176 -28
tests/entrypoints/openai/rpc/__init__.py
tests/entrypoints/openai/rpc/test_zmq_client.py new
+119
@@ -0,0 +1,119 @@
1 +import asyncio
2 +import tempfile
3 +import unittest
4 +import unittest.mock
5 +import uuid
6 +
7 +import pytest
8 +import pytest_asyncio
9 +
10 +from vllm.engine.async_llm_engine import AsyncLLMEngine
11 +from vllm.entrypoints.openai.rpc.client import (AsyncEngineRPCClient,
12 + RPCClientClosedError)
13 +from vllm.entrypoints.openai.rpc.server import AsyncEngineRPCServer
14 +
15 +
16 +@pytest.fixture(scope="function")
17 +def tmp_socket():
18 + with tempfile.TemporaryDirectory() as td:
19 + yield f"ipc://{td}/{uuid.uuid4()}"
20 +
21 +
22 +@pytest_asyncio.fixture(scope="function")
23 +async def dummy_server(tmp_socket, monkeypatch):
24 + dummy_engine = unittest.mock.AsyncMock()
25 +
26 + def dummy_engine_builder(*args, **kwargs):
27 + return dummy_engine
28 +
29 + with monkeypatch.context() as m:
30 + m.setattr(AsyncLLMEngine, "from_engine_args", dummy_engine_builder)
31 + server = AsyncEngineRPCServer(None, None, rpc_path=tmp_socket)
32 +
33 + loop = asyncio.get_running_loop()
34 + server_task = loop.create_task(server.run_server_loop())
35 +
36 + try:
37 + yield server
38 + finally:
39 + server_task.cancel()
40 + server.cleanup()
41 +
42 +
43 +@pytest_asyncio.fixture(scope="function")
44 +async def client(tmp_socket):
45 + client = AsyncEngineRPCClient(rpc_path=tmp_socket)
46 + # Sanity check: the server is connected
47 + await client._wait_for_server_rpc()
48 +
49 + try:
50 + yield client
51 + finally:
52 + client.close()
53 +
54 +
55 +@pytest.mark.asyncio
56 +async def test_client_data_methods_use_timeouts(monkeypatch, dummy_server,
57 + client: AsyncEngineRPCClient):
58 + with monkeypatch.context() as m:
59 + # Make the server _not_ reply with a model config
60 + m.setattr(dummy_server, "get_config", lambda x: None)
61 + m.setattr(client, "_data_timeout", 10)
62 +
63 + # And ensure the task completes anyway
64 + # (client.setup() invokes server.get_config())
65 + client_task = asyncio.get_running_loop().create_task(client.setup())
66 + with pytest.raises(TimeoutError, match="Server didn't reply within"):
67 + await asyncio.wait_for(client_task, timeout=0.05)
68 +
69 +
70 +@pytest.mark.asyncio
71 +async def test_client_aborts_use_timeouts(monkeypatch, dummy_server,
72 + client: AsyncEngineRPCClient):
73 + with monkeypatch.context() as m:
74 + # Hang all abort requests
75 + m.setattr(dummy_server, "abort", lambda x: None)
76 + m.setattr(client, "_data_timeout", 10)
77 +
78 + # Ensure the client doesn't hang
79 + client_task = asyncio.get_running_loop().create_task(
80 + client.abort("test request id"))
81 + with pytest.raises(TimeoutError, match="Server didn't reply within"):
82 + await asyncio.wait_for(client_task, timeout=0.05)
83 +
84 +
85 +@pytest.mark.asyncio
86 +async def test_client_data_methods_reraise_exceptions(
87 + monkeypatch, dummy_server, client: AsyncEngineRPCClient):
88 + with monkeypatch.context() as m:
89 + # Make the server raise some random exception
90 + exception = RuntimeError("Client test exception")
91 +
92 + def raiser():
93 + raise exception
94 +
95 + m.setattr(dummy_server.engine, "get_model_config", raiser)
96 + m.setattr(client, "_data_timeout", 10)
97 +
98 + client_task = asyncio.get_running_loop().create_task(client.setup())
99 + # And ensure the task completes, raising the exception
100 + with pytest.raises(RuntimeError, match=str(exception)):
101 + await asyncio.wait_for(client_task, timeout=0.05)
102 +
103 +
104 +@pytest.mark.asyncio
105 +async def test_client_errors_after_closing(monkeypatch, dummy_server,
106 + client: AsyncEngineRPCClient):
107 +
108 + client.close()
109 +
110 + # Healthchecks and generate requests will fail with explicit errors
111 + with pytest.raises(RPCClientClosedError):
112 + await client.check_health()
113 + with pytest.raises(RPCClientClosedError):
114 + async for _ in client.generate(None, None, None):
115 + pass
116 +
117 + # But no-ops like aborting will pass
118 + await client.abort("test-request-id")
119 + await client.do_log_stats()
vllm/entrypoints/openai/api_server.py
+3 -2
@@ -6,7 +6,7 @@ import os
6 import re
7 import tempfile
8 from argparse import Namespace
9 -from contextlib import asynccontextmanager
9 +from contextlib import asynccontextmanager, suppress
10 from http import HTTPStatus
11 from typing import AsyncIterator, Optional, Set
12
@@ -83,7 +83,8 @@ async def lifespan(app: FastAPI):
83 async def _force_log():
84 while True:
85 await asyncio.sleep(10)
86 - await async_engine_client.do_log_stats()
86 + with suppress(Exception):
87 + await async_engine_client.do_log_stats()
88
89 if not engine_args.disable_log_stats:
90 task = asyncio.create_task(_force_log())
vllm/entrypoints/openai/rpc/__init__.py
-4
@@ -10,10 +10,6 @@ from vllm.sampling_params import SamplingParams
10 # Success string used for RPC instructions.
11 VLLM_RPC_SUCCESS_STR = "SUCCESS"
12
13 -# Timeouts.
14 -VLLM_RPC_SERVER_START_TIMEOUT_MS = 1000
15 -VLLM_RPC_HEALTH_TIMEOUT_MS = 10000
16 -
13 # Minimum value of ZMQ.SOCKET_LIMIT to run mp.
14 VLLM_RPC_SOCKET_LIMIT_CUTOFF = 2000
15
vllm/entrypoints/openai/rpc/client.py
+48 -22
@@ -1,5 +1,5 @@
1 import asyncio
2 -from contextlib import contextmanager
2 +from contextlib import contextmanager, suppress
3 from typing import Any, AsyncGenerator, Mapping, Optional
4 from uuid import uuid4
5
@@ -11,13 +11,12 @@ from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11 ParallelConfig, SchedulerConfig)
12 # yapf: disable
13 from vllm.entrypoints.openai.rpc import (RPC_REQUEST_TYPE,
14 - VLLM_RPC_HEALTH_TIMEOUT_MS,
15 - VLLM_RPC_SERVER_START_TIMEOUT_MS,
14 VLLM_RPC_SOCKET_LIMIT_CUTOFF,
15 VLLM_RPC_SUCCESS_STR,
16 VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
17 RPCGenerateRequest, RPCUtilityRequest)
18 # yapf: enable
19 +from vllm.envs import VLLM_RPC_GET_DATA_TIMEOUT_MS
20 from vllm.inputs import PromptInputs
21 from vllm.logger import init_logger
22 from vllm.lora.request import LoRARequest
@@ -32,6 +31,17 @@ logger = init_logger(__name__)
31 INPROC_PROXY_PATH = f"inproc://{uuid4()}"
32
33
34 +class RPCClientClosedError(Exception):
35 + """Exception class raised when the client is used post-close.
36 +
37 + The client can be closed, which closes the ZMQ context. This normally
38 + happens on server shutdown. In some cases, methods like abort and
39 + do_log_stats will still be called and then try to open a socket, which
40 + causes a ZMQError and creates a huge stack trace.
41 + So, we throw this error such that we can suppress it.
42 + """
43 +
44 +
45 class AsyncEngineRPCClient:
46 """
47 RPCClient that connects to the RPCServer wrapping AsyncLLMEngine.
@@ -85,6 +95,8 @@ class AsyncEngineRPCClient:
95
96 def __init__(self, rpc_path: str):
97 self.context = zmq.asyncio.Context()
98 + self._data_timeout = VLLM_RPC_GET_DATA_TIMEOUT_MS
99 + self._errored = False
100
101 # Maximum number of sockets that can be opened (typically 65536).
102 # ZMQ_SOCKET_LIMIT (http://api.zeromq.org/4-2:zmq-ctx-get)
@@ -143,7 +155,6 @@ class AsyncEngineRPCClient:
155
156 # Wait until server is ready.
157 await self._wait_for_server_rpc()
146 - self._errored = False
158
159 # Get the configs.
160 self.model_config = await self._get_model_config_rpc()
@@ -170,6 +181,15 @@ class AsyncEngineRPCClient:
181 @contextmanager
182 def to_proxy_socket(self):
183 # Connect to the RPCServer via the proxy.
184 +
185 + # Raise a sensible error if the client was already closed.
186 + # This can happen if a server shutdown is triggered but some coroutines
187 + # are still running requests.
188 + # There should not be a race condition with this check because we don't
189 + # yield to the event loop between here and opening the socket.
190 + if self.context.closed:
191 + raise RPCClientClosedError("The ZMQ client has already shut down")
192 +
193 # Note that we use DEALER to enable asynchronous communication
194 # to enable streaming.
195 socket = self.context.socket(zmq.constants.DEALER)
@@ -189,9 +209,18 @@ class AsyncEngineRPCClient:
209 # Ping RPCServer with a request.
210 await socket.send_multipart([cloudpickle.dumps(request)])
211
212 + # Make sure the server responds
213 + if await socket.poll(timeout=self._data_timeout) == 0:
214 + raise TimeoutError("Server didn't reply within "
215 + f"{self._data_timeout} ms")
216 +
217 # Await the data from the Server.
218 data = cloudpickle.loads(await socket.recv())
219
220 + if isinstance(data, Exception):
221 + # Re-raise exceptions returned by the server
222 + raise data
223 +
224 if not isinstance(data, expected_type):
225 # LoRAConfig can be None.
226 if expected_type == LoRAConfig and data is None:
@@ -208,29 +237,28 @@ class AsyncEngineRPCClient:
237 self,
238 request: RPC_REQUEST_TYPE,
239 error_message: str,
211 - timeout: Optional[int] = None,
240 socket: Optional[zmq.asyncio.Socket] = None):
241 """Send one-way RPC request to trigger an action."""
242
243 async def do_rpc_call(socket: zmq.asyncio.Socket,
216 - request: RPC_REQUEST_TYPE,
217 - timeout=None):
244 + request: RPC_REQUEST_TYPE):
245
246 await socket.send_multipart([cloudpickle.dumps(request)])
247
221 - if timeout is not None and await socket.poll(timeout=timeout) == 0:
222 - raise TimeoutError(f"Server didn't reply within {timeout} ms")
248 + if await socket.poll(timeout=self._data_timeout) == 0:
249 + raise TimeoutError("Server didn't reply within "
250 + f"{self._data_timeout} ms")
251
252 return cloudpickle.loads(await socket.recv())
253
254 # Make a new socket connection.
255 if socket is None:
256 with self.to_proxy_socket() as socket:
229 - response = await do_rpc_call(socket, request, timeout)
257 + response = await do_rpc_call(socket, request)
258
259 # Use existing socket connection.
260 else:
233 - response = await do_rpc_call(socket, request, timeout)
261 + response = await do_rpc_call(socket, request)
262
263 if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
264 if isinstance(response, Exception):
@@ -255,8 +283,7 @@ class AsyncEngineRPCClient:
283
284 await self._send_one_way_rpc_request(
285 request=RPCUtilityRequest.IS_SERVER_READY,
258 - error_message="Unable to start RPC Server",
259 - timeout=VLLM_RPC_SERVER_START_TIMEOUT_MS)
286 + error_message="Unable to start RPC Server")
287
288 async def _get_model_config_rpc(self) -> ModelConfig:
289 """Get the ModelConfig object from the RPC Server"""
@@ -308,17 +335,17 @@ class AsyncEngineRPCClient:
335
336 async def abort(self, request_id: str):
337 """Send an ABORT_REQUEST signal to the RPC Server"""
311 -
312 - await self._send_one_way_rpc_request(
313 - request=RPCAbortRequest(request_id),
314 - error_message=f"RPCAbortRequest {request_id} failed")
338 + with suppress(RPCClientClosedError):
339 + await self._send_one_way_rpc_request(
340 + request=RPCAbortRequest(request_id),
341 + error_message=f"RPCAbortRequest {request_id} failed")
342
343 async def do_log_stats(self):
344 """Send a DO_LOG_STATS signal to the RPC Server"""
318 -
319 - await self._send_one_way_rpc_request(
320 - request=RPCUtilityRequest.DO_LOG_STATS,
321 - error_message="RPCRequest DO_LOG_STATS failed.")
345 + with suppress(RPCClientClosedError):
346 + await self._send_one_way_rpc_request(
347 + request=RPCUtilityRequest.DO_LOG_STATS,
348 + error_message="RPCRequest DO_LOG_STATS failed.")
349
350 @property
351 def is_running(self) -> bool:
@@ -393,7 +420,6 @@ class AsyncEngineRPCClient:
420 await self._send_one_way_rpc_request(
421 request=RPCUtilityRequest.IS_SERVER_HEALTHY,
422 error_message="Got Unhealthy response from RPC Server",
396 - timeout=VLLM_RPC_HEALTH_TIMEOUT_MS,
423 socket=socket)
424
425 async def encode(self, *args,
vllm/envs.py
+6
@@ -56,6 +56,7 @@ if TYPE_CHECKING:
56 VERBOSE: bool = False
57 VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
58 VLLM_TEST_FORCE_FP8_MARLIN: bool = False
59 + VLLM_RPC_GET_DATA_TIMEOUT_MS: int = 5000
60 VLLM_ALLOW_ENGINE_USE_RAY: bool = False
61 VLLM_PLUGINS: Optional[List[str]] = None
62 VLLM_TORCH_PROFILER_DIR: Optional[str] = None
@@ -374,6 +375,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
375 (os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
376 ("1", "true")),
377
378 + # Time in ms for the zmq client to wait for a response from the backend
379 + # server for simple data operations
380 + "VLLM_RPC_GET_DATA_TIMEOUT_MS":
381 + lambda: int(os.getenv("VLLM_RPC_GET_DATA_TIMEOUT_MS", "5000")),
382 +
383 # If set, allow running the engine as a separate ray actor,
384 # which is a deprecated feature soon to be removed.
385 # See https://github.com/vllm-project/vllm/issues/7045