@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fde47d3bc2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[BugFix] Fix frontend multiprocessing hang (#7217)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>

Maximilien de Bayser committed Aug 7, 2024 at 15:09 UTC fde47d3bc2d51fa02d1132d541de52924a0b0838
3 files changed +67 -5
tests/entrypoints/openai/test_mp_crash.py new
+35
@@ -0,0 +1,35 @@
1 +from typing import Any
2 +
3 +import pytest
4 +
5 +from vllm.engine.async_llm_engine import AsyncLLMEngine
6 +from vllm.entrypoints.openai.api_server import build_async_engine_client
7 +from vllm.entrypoints.openai.cli_args import make_arg_parser
8 +from vllm.utils import FlexibleArgumentParser
9 +
10 +
11 +def crashing_from_engine_args(
12 + cls,
13 + engine_args: Any = None,
14 + start_engine_loop: Any = None,
15 + usage_context: Any = None,
16 + stat_loggers: Any = None,
17 +) -> "AsyncLLMEngine":
18 + raise Exception("foo")
19 +
20 +
21 +@pytest.mark.asyncio
22 +async def test_mp_crash_detection(monkeypatch):
23 +
24 + with pytest.raises(RuntimeError) as excinfo, monkeypatch.context() as m:
25 + m.setattr(AsyncLLMEngine, "from_engine_args",
26 + crashing_from_engine_args)
27 + parser = FlexibleArgumentParser(
28 + description="vLLM's remote OpenAI server.")
29 + parser = make_arg_parser(parser)
30 + args = parser.parse_args([])
31 +
32 + async with build_async_engine_client(args):
33 + pass
34 + assert "The server process died before responding to the readiness probe"\
35 + in str(excinfo.value)
vllm/entrypoints/openai/api_server.py
+10 -1
@@ -120,9 +120,18 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
120
121 # Build RPCClient, which conforms to AsyncEngineClient Protocol.
122 async_engine_client = AsyncEngineRPCClient(rpc_path)
123 - await async_engine_client.setup()
123
124 try:
125 + while True:
126 + try:
127 + await async_engine_client.setup()
128 + break
129 + except TimeoutError as e:
130 + if not rpc_server_process.is_alive():
131 + raise RuntimeError(
132 + "The server process died before "
133 + "responding to the readiness probe") from e
134 +
135 yield async_engine_client
136 finally:
137 # Ensure rpc server process was terminated
vllm/entrypoints/openai/rpc/client.py
+22 -4
@@ -18,6 +18,9 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
18 from vllm.sampling_params import SamplingParams
19 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
20
21 +# Time to wait before checking it the server process is alive.
22 +SERVER_START_TIMEOUT_MS = 1000
23 +
24
25 class AsyncEngineRPCClient:
26
@@ -61,7 +64,16 @@ class AsyncEngineRPCClient:
64 socket.connect(self.rpc_path)
65 yield socket
66 finally:
64 - socket.close()
67 + # linger == 0 means discard unsent messages
68 + # when the socket is closed. This is necessary
69 + # because otherwise self.context.destroy() will
70 + # wait for 30 seconds until unsent messages are
71 + # received, which is impossible if the server
72 + # crashed. In the absence of a server crash we
73 + # always expect a response before closing the
74 + # socket anyway.
75 + # Reference: http://api.zeromq.org/4-2:zmq-setsockopt#toc24
76 + socket.close(linger=0)
77
78 async def _send_get_data_rpc_request(self, request: RPCUtilityRequest,
79 expected_type: Any,
@@ -85,14 +97,19 @@ class AsyncEngineRPCClient:
97
98 return data
99
88 - async def _send_one_way_rpc_request(self, request: RPC_REQUEST_TYPE,
89 - error_message: str):
100 + async def _send_one_way_rpc_request(self,
101 + request: RPC_REQUEST_TYPE,
102 + error_message: str,
103 + timeout: Optional[int] = None):
104 """Send one-way RPC request to trigger an action."""
105 with self.socket() as socket:
106 # Ping RPC Server with request.
107 await socket.send(cloudpickle.dumps(request))
108
109 # Await acknowledgement from RPCServer.
110 + if timeout is not None and await socket.poll(timeout=timeout) == 0:
111 + raise TimeoutError(f"server didn't reply within {timeout} ms")
112 +
113 response = cloudpickle.loads(await socket.recv())
114
115 if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
@@ -117,7 +134,8 @@ class AsyncEngineRPCClient:
134
135 await self._send_one_way_rpc_request(
136 request=RPCUtilityRequest.IS_SERVER_READY,
120 - error_message="Unable to start RPC Server.")
137 + error_message="Unable to start RPC Server.",
138 + timeout=SERVER_START_TIMEOUT_MS)
139
140 async def _get_model_config_rpc(self) -> ModelConfig:
141 """Get the ModelConfig object from the RPC Server"""