A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix][Frontend] Fix Issues Under High Load With `zeromq` Frontend (#7394)
Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Robert Shaw committed
Aug 21, 2024 at 13:34 UTC
f7e3b0c5aa2862d27b8872084c5ca934659ceef8
9 files changed
+322
-141
.buildkite/test-pipeline.yaml
+1
@@ -86,6 +86,7 @@ steps:
86
- vllm/
87
commands:
88
- pip install -e ./plugins/vllm_add_dummy_model
89
+ - pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@a4987bba6e9e9b3f22bd3a6c1ecf0abd04fd5622#egg=lm_eval[api]
90
- pytest -v -s entrypoints/llm
91
- pytest -v -s entrypoints/openai
92
tests/entrypoints/openai/test_accuracy.py
new
+55
@@ -0,0 +1,55 @@
1
+"""
2
+This file test accuracy of the vLLM server via LMEval.
3
+It uses local-completions, which interacts with vLLM
4
+through the OAI API with N concurrent connections.
5
+This simulates real work usage of the API and makes
6
+sure that the zmq frontend mp RPC message passing and
7
+AsyncLLMEngine are working correctly.
8
+"""
9
+
10
+import lm_eval
11
+import pytest
12
+
13
+from ...utils import RemoteOpenAIServer
14
+
15
+MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
16
+NUM_CONCURRENT = 500
17
+TASK = "gsm8k"
18
+FILTER = "exact_match,strict-match"
19
+RTOL = 0.03
20
+EXPECTED_VALUE = 0.58
21
+
22
+
23
+@pytest.fixture(scope="module")
24
+def server():
25
+ args = [
26
+ "--max-model-len", "4096", "--enable-chunked-prefill",
27
+ "--disable-log-requests", "--enforce-eager"
28
+ ]
29
+
30
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
31
+ yield remote_server
32
+
33
+
34
+@pytest.fixture(scope="module")
35
+def server_data(server):
36
+ return {
37
+ "url": f"{server.url_for('v1')}/completions",
38
+ }
39
+
40
+
41
+def test_lm_eval_accuracy(server_data):
42
+ model_args = (f"model={MODEL_NAME},"
43
+ f"base_url={server_data['url']},"
44
+ f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False")
45
+
46
+ results = lm_eval.simple_evaluate(
47
+ model="local-completions",
48
+ model_args=model_args,
49
+ tasks=TASK,
50
+ )
51
+
52
+ measured_value = results["results"][TASK][FILTER]
53
+ assert (measured_value - RTOL < EXPECTED_VALUE
54
+ and measured_value + RTOL > EXPECTED_VALUE
55
+ ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
vllm/engine/async_llm_engine.py
+5
@@ -766,6 +766,11 @@ class AsyncLLMEngine:
766
def errored(self) -> bool:
767
return self._errored_with is not None
768
769
+ @property
770
+ def limit_concurrency(self) -> Optional[int]:
771
+ """Maximum number of concurrently running requests."""
772
+ return None
773
+
774
def set_errored(self, exc: Exception) -> None:
775
self._errored_with = exc
776
vllm/engine/protocol.py
+4
@@ -29,6 +29,10 @@ class AsyncEngineClient(Protocol):
29
def errored(self) -> bool:
30
...
31
32
+ @property
33
+ def limit_concurrency(self) -> Optional[int]:
34
+ """Maximum number of concurrently running requests."""
35
+
36
def generate(
37
self,
38
inputs: PromptInputs,
vllm/entrypoints/launcher.py
+9
@@ -27,6 +27,15 @@ async def serve_http(app: FastAPI, engine: AsyncEngineClient,
27
28
logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
29
30
+ # Set concurrency limits in uvicorn if running in multiprocessing mode
31
+ # since zmq has maximum socket limit of zmq.constants.SOCKET_LIMIT (65536).
32
+ if engine.limit_concurrency is not None:
33
+ logger.info(
34
+ "Launching Uvicorn with --limit_concurrency %s. To avoid this "
35
+ "limit at the expense of performance run with "
36
+ "--disable-frontend-multiprocessing", engine.limit_concurrency)
37
+ uvicorn_kwargs["limit_concurrency"] = engine.limit_concurrency
38
+
39
config = uvicorn.Config(app, **uvicorn_kwargs)
40
server = uvicorn.Server(config)
41
_add_shutdown_handlers(app, server, engine)
vllm/entrypoints/openai/api_server.py
+6
-5
@@ -135,6 +135,12 @@ async def build_async_engine_client(
135
logger.info("Multiprocessing frontend to use %s for RPC Path.",
136
rpc_path)
137
138
+ # Build RPCClient, which conforms to AsyncEngineClient Protocol.
139
+ # NOTE: Actually, this is not true yet. We still need to support
140
+ # embedding models via RPC (see TODO above)
141
+ rpc_client = AsyncEngineRPCClient(rpc_path)
142
+ async_engine_client = rpc_client # type: ignore
143
+
144
# Start RPCServer in separate process (holds the AsyncLLMEngine).
145
context = multiprocessing.get_context("spawn")
146
# the current process might have CUDA context,
@@ -145,11 +151,6 @@ async def build_async_engine_client(
151
rpc_server_process.start()
152
logger.info("Started engine process with PID %d",
153
rpc_server_process.pid)
148
- # Build RPCClient, which conforms to AsyncEngineClient Protocol.
149
- # NOTE: Actually, this is not true yet. We still need to support
150
- # embedding models via RPC (see TODO above)
151
- rpc_client = AsyncEngineRPCClient(rpc_path)
152
- async_engine_client = rpc_client # type: ignore
154
155
try:
156
while True:
vllm/entrypoints/openai/rpc/__init__.py
+12
-2
@@ -7,8 +7,18 @@ from vllm.lora.request import LoRARequest
7
from vllm.prompt_adapter.request import PromptAdapterRequest
8
from vllm.sampling_params import SamplingParams
9
10
+# Success string used for RPC instructions.
11
VLLM_RPC_SUCCESS_STR = "SUCCESS"
11
-VLLM_RPC_HEALTHY_STR = "HEALTHY"
12
+
13
+# Timeouts.
14
+VLLM_RPC_SERVER_START_TIMEOUT_MS = 1000
15
+VLLM_RPC_HEALTH_TIMEOUT_MS = 10000
16
+
17
+# Minimum value of ZMQ.SOCKET_LIMIT to run mp.
18
+VLLM_RPC_SOCKET_LIMIT_CUTOFF = 2000
19
+
20
+# HWM is set to Infinity.
21
+VLLM_RPC_ZMQ_HWM = 0
22
23
24
@dataclass
@@ -34,7 +44,7 @@ class RPCUtilityRequest(Enum):
44
GET_SCHEDULER_CONFIG = 5
45
GET_LORA_CONFIG = 6
46
DO_LOG_STATS = 7
37
- CHECK_HEALTH = 8
47
+ IS_SERVER_HEALTHY = 8
48
IS_TRACING_ENABLED = 9
49
50
vllm/entrypoints/openai/rpc/client.py
+180
-68
@@ -1,5 +1,7 @@
1
+import asyncio
2
from contextlib import contextmanager
3
from typing import Any, AsyncGenerator, Mapping, Optional
4
+from uuid import uuid4
5
6
import cloudpickle
7
import zmq
@@ -7,32 +9,140 @@ import zmq.asyncio
9
10
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11
ParallelConfig, SchedulerConfig)
12
+# yapf: disable
13
from vllm.entrypoints.openai.rpc import (RPC_REQUEST_TYPE,
11
- VLLM_RPC_HEALTHY_STR,
12
- VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
14
+ VLLM_RPC_HEALTH_TIMEOUT_MS,
15
+ VLLM_RPC_SERVER_START_TIMEOUT_MS,
16
+ VLLM_RPC_SOCKET_LIMIT_CUTOFF,
17
+ VLLM_RPC_SUCCESS_STR,
18
+ VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
19
RPCGenerateRequest, RPCUtilityRequest)
20
+# yapf: enable
21
from vllm.inputs import PromptInputs
22
+from vllm.logger import init_logger
23
from vllm.lora.request import LoRARequest
24
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
25
from vllm.prompt_adapter.request import PromptAdapterRequest
26
from vllm.sampling_params import SamplingParams
27
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
28
21
-# Time to wait before checking it the server process is alive.
22
-SERVER_START_TIMEOUT_MS = 1000
29
+logger = init_logger(__name__)
30
+
31
+# Path used for inprocess proxy.
32
+INPROC_PROXY_PATH = f"inproc://{uuid4()}"
33
34
35
class AsyncEngineRPCClient:
36
+ """
37
+ RPCClient that connects to the RPCServer wrapping AsyncLLMEngine.
38
+
39
+ The overall design mirrors the Asynchronous Client Server Pattern
40
+ https://zguide.zeromq.org/docs/chapter3/#The-Asynchronous-Client-Server-Pattern
41
+
42
+ On startup, the RPCClient:
43
+ - makes DEALER socket (to_rpc_server) that connects to the RPCServer
44
+ via ipc, which uses unix sockets under the hood
45
+ (https://libzmq.readthedocs.io/en/zeromq4-1/zmq_ipc.html)
46
+ - makes ROUTER socket (from_api_server) that binds to a random
47
+ inproc address, which uses memory under the hood
48
+ (https://libzmq.readthedocs.io/en/zeromq3-x/zmq_inproc.html)
49
+ - runs a proxy in a background asyncio task between
50
+ from_api_server (ROUTER, inproc) and to_rpc_server (DEALER ipc, )
51
+
52
+ Each request handled by the asyncio api_server calls generate():
53
+ - make a DEALER socket that connects to from_api_server via inproc
54
+ - send a RCPGenerateRequest to the inproc socket
55
+ - background proxy forwards the request from inproc -> ipc
56
+ - RPCServer responds to the request one token at a time over ipc
57
+ - background proxy forwards the response from ipc -> inproc
58
+
59
+ The connection looks like this:
60
+ DEALER <- inproc -> [ ROUTER | DEALER ] <- ipc -> DEALER
61
+
62
+ Message routing is performed via identities that are managed by the
63
+ ROUTER socket. ROUTER sockets track every connection it has and
64
+ tells the caller about these. The way it tells the caller is to stick
65
+ the connection identity in front of each message received. When we
66
+ send the message via a ROUTER, we first send an identity frame.
67
+ See https://zguide.zeromq.org/docs/chapter3/#The-Extended-Reply-Envelope
68
+ for more details on connection identities.
69
+
70
+ This proxy design enables us to use a single unix socket, which
71
+ improves performance by avoiding syscalls (~5%) and avoids resource limits
72
+ such as ulimit, which defaults to 1024 on ubuntu.
73
+
74
+ Note: we run set_hwm(0) on each socket, which sets the HWM to inf,
75
+ which is required to avoid dropping messages under high load.
76
+ This is generally not advisable. However, since we are in control
77
+ of both sides of the connection + failure on either side is
78
+ catastrophic to the overall system health and memory profiling
79
+ suggests limited memory overhead relative to asyncio, we will
80
+ proceed for now.
81
+
82
+ See https://zguide.zeromq.org/docs/chapter2/#High-Water-Marks
83
+ for more details on high water marks.
84
+ """
85
86
def __init__(self, rpc_path: str):
87
self.context = zmq.asyncio.Context()
29
- self.rpc_path = rpc_path
88
+
89
+ # Maximum number of sockets that can be opened (typically 65536).
90
+ # ZMQ_SOCKET_LIMIT (http://api.zeromq.org/4-2:zmq-ctx-get)
91
+ socket_limit = self.context.get(zmq.constants.SOCKET_LIMIT)
92
+ if socket_limit < VLLM_RPC_SOCKET_LIMIT_CUTOFF:
93
+ raise ValueError(
94
+ f"Found zmq.constants.SOCKET_LIMIT={socket_limit}, which caps "
95
+ "the number of concurrent requests vLLM can process. Launch "
96
+ "vLLM with --disable-frontend-multiprocessing and open a "
97
+ "GitHub issue so we can investigate.")
98
+
99
+ # We only have 1 ipc connection that uses unix sockets, so
100
+ # safe to set MAX_SOCKETS to the zmq SOCKET_LIMIT (i.e. will
101
+ # not run into ulimit issues)
102
+ self.context.set(zmq.constants.MAX_SOCKETS, socket_limit)
103
+
104
+ # IPC connection to RPC Server (uses unix sockets).
105
+ self.to_rpc_server = self.context.socket(zmq.constants.DEALER)
106
+ self.to_rpc_server.set_hwm(VLLM_RPC_ZMQ_HWM)
107
+ self.to_rpc_server.bind(rpc_path)
108
+
109
+ # In process proxy to RPC Server (uses memory-based messaging).
110
+ self.from_api_server = self.context.socket(zmq.constants.ROUTER)
111
+ self.from_api_server.set_hwm(VLLM_RPC_ZMQ_HWM)
112
+ self.from_api_server.bind(INPROC_PROXY_PATH)
113
+
114
+ # Asyncio background task for the proxy.
115
+ self.proxy_task = asyncio.create_task(
116
+ self.run_proxy(self.from_api_server, self.to_rpc_server))
117
+
118
+ # Since we open 1 inproc socket per request, we have a hard cap on
119
+ # the number of requests that can run in vLLM w. frontend
120
+ # mulitprocessing. This value is used uvicorn to launch
121
+ # with --limit-concurrency to return 503 when server is overloaded.
122
+ # We need 2 sockets per request - 2:
123
+ # 1 for generate(), 1 for abort(), do_log_stats(), check_health()
124
+ self.limit_concurrency = socket_limit // 2 - 2
125
+
126
+ async def run_proxy(self, socket_from, socket_to):
127
+ """Background task that runs a proxy"""
128
+ poller = zmq.asyncio.Poller()
129
+ poller.register(socket_from, zmq.constants.POLLIN)
130
+ poller.register(socket_to, zmq.constants.POLLIN)
131
+ while True:
132
+ events = await poller.poll()
133
+ events = dict(events)
134
+ if socket_from in events:
135
+ identity, msg = await socket_from.recv_multipart()
136
+ await socket_to.send_multipart([identity, msg])
137
+ if socket_to in events:
138
+ identity, msg = await socket_to.recv_multipart()
139
+ await socket_from.send_multipart([identity, msg])
140
141
async def setup(self):
142
"""Setup the client before it starts sending server requests."""
143
144
# Wait until server is ready.
35
- await self.wait_for_server()
145
+ await self._wait_for_server_rpc()
146
self._errored = False
147
148
# Get the configs.
@@ -51,29 +161,23 @@ class AsyncEngineRPCClient:
161
162
def close(self):
163
"""Destroy the ZeroMQ Context."""
164
+ # Close all sockets associated with this context and
165
+ # then terminate the context.
166
+ self.from_api_server.close()
167
+ self.to_rpc_server.close()
168
self.context.destroy()
169
170
@contextmanager
57
- def socket(self):
58
- # Ensure client sockets are always closed after use
59
-
60
- # Connect to RPC socket for Request-Reply pattern,
171
+ def to_proxy_socket(self):
172
+ # Connect to the RPCServer via the proxy.
173
# Note that we use DEALER to enable asynchronous communication
174
# to enable streaming.
175
socket = self.context.socket(zmq.constants.DEALER)
176
+ socket.set_hwm(VLLM_RPC_ZMQ_HWM)
177
try:
65
- socket.connect(self.rpc_path)
178
+ socket.connect(INPROC_PROXY_PATH)
179
yield socket
180
finally:
68
- # linger == 0 means discard unsent messages
69
- # when the socket is closed. This is necessary
70
- # because otherwise self.context.destroy() will
71
- # wait for 30 seconds until unsent messages are
72
- # received, which is impossible if the server
73
- # crashed. In the absence of a server crash we
74
- # always expect a response before closing the
75
- # socket anyway.
76
- # Reference: http://api.zeromq.org/4-2:zmq-setsockopt#toc24
181
socket.close(linger=0)
182
183
async def _send_get_data_rpc_request(self, request: RPCUtilityRequest,
@@ -81,10 +185,9 @@ class AsyncEngineRPCClient:
185
error_message: str) -> Any:
186
"""Send an RPC request that is expecting data back."""
187
84
- with self.socket() as socket:
85
-
188
+ with self.to_proxy_socket() as socket:
189
# Ping RPCServer with a request.
87
- await socket.send(cloudpickle.dumps(request))
190
+ await socket.send_multipart([cloudpickle.dumps(request)])
191
192
# Await the data from the Server.
193
data = cloudpickle.loads(await socket.recv())
@@ -93,31 +196,48 @@ class AsyncEngineRPCClient:
196
# LoRAConfig can be None.
197
if expected_type == LoRAConfig and data is None:
198
pass
199
+ elif isinstance(data, Exception):
200
+ logger.error(error_message)
201
+ raise data
202
else:
203
raise ValueError(error_message)
204
205
return data
206
101
- async def _send_one_way_rpc_request(self,
102
- request: RPC_REQUEST_TYPE,
103
- error_message: str,
104
- timeout: Optional[int] = None):
207
+ async def _send_one_way_rpc_request(
208
+ self,
209
+ request: RPC_REQUEST_TYPE,
210
+ error_message: str,
211
+ timeout: Optional[int] = None,
212
+ socket: Optional[zmq.asyncio.Socket] = None):
213
"""Send one-way RPC request to trigger an action."""
106
- with self.socket() as socket:
107
- # Ping RPC Server with request.
108
- await socket.send(cloudpickle.dumps(request))
214
110
- # Await acknowledgement from RPCServer.
215
+ async def do_rpc_call(socket: zmq.asyncio.Socket,
216
+ request: RPC_REQUEST_TYPE,
217
+ timeout=None):
218
+
219
+ await socket.send_multipart([cloudpickle.dumps(request)])
220
+
221
if timeout is not None and await socket.poll(timeout=timeout) == 0:
112
- raise TimeoutError(f"server didn't reply within {timeout} ms")
222
+ raise TimeoutError(f"Server didn't reply within {timeout} ms")
223
+
224
+ return cloudpickle.loads(await socket.recv())
225
114
- response = cloudpickle.loads(await socket.recv())
226
+ # Make a new socket connection.
227
+ if socket is None:
228
+ with self.to_proxy_socket() as socket:
229
+ response = await do_rpc_call(socket, request, timeout)
230
+
231
+ # Use existing socket connection.
232
+ else:
233
+ response = await do_rpc_call(socket, request, timeout)
234
235
if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
236
+ if isinstance(response, Exception):
237
+ logger.error(error_message)
238
+ raise response
239
raise ValueError(error_message)
240
119
- return response
120
-
241
async def get_tokenizer(self, lora_request: LoRARequest):
242
return await self.tokenizer.get_lora_tokenizer_async(lora_request)
243
@@ -130,13 +250,13 @@ class AsyncEngineRPCClient:
250
async def is_tracing_enabled(self) -> bool:
251
return self.tracing_flag
252
133
- async def wait_for_server(self):
253
+ async def _wait_for_server_rpc(self):
254
"""Wait for the RPCServer to start up."""
255
256
await self._send_one_way_rpc_request(
257
request=RPCUtilityRequest.IS_SERVER_READY,
138
- error_message="Unable to start RPC Server.",
139
- timeout=SERVER_START_TIMEOUT_MS)
258
+ error_message="Unable to start RPC Server",
259
+ timeout=VLLM_RPC_SERVER_START_TIMEOUT_MS)
260
261
async def _get_model_config_rpc(self) -> ModelConfig:
262
"""Get the ModelConfig object from the RPC Server"""
@@ -184,8 +304,7 @@ class AsyncEngineRPCClient:
304
return await self._send_get_data_rpc_request(
305
RPCUtilityRequest.IS_TRACING_ENABLED,
306
expected_type=bool,
187
- error_message="Could not get is_tracing_enabled flag from RPC "
188
- "Server")
307
+ error_message="Could not get is_tracing_enabled from RPC Server")
308
309
async def abort(self, request_id: str):
310
"""Send an ABORT_REQUEST signal to the RPC Server"""
@@ -226,8 +345,7 @@ class AsyncEngineRPCClient:
345
346
finished = False
347
try:
229
- with self.socket() as socket:
230
-
348
+ with self.to_proxy_socket() as socket:
349
# Send RPCGenerateRequest to the RPCServer.
350
await socket.send_multipart([
351
cloudpickle.dumps(
@@ -246,43 +364,37 @@ class AsyncEngineRPCClient:
364
request_output = cloudpickle.loads(message)
365
366
if isinstance(request_output, Exception):
249
- # On exception, check if the server is still healthy.
250
- # Use this to set the sync `is_running` and `errored`
251
- # properties.
252
- try:
253
- await self.check_health()
254
- except Exception:
255
- self._errored = True
367
+ # On exception, check if the server is still healthy
368
+ # possibly setting the `errored` property.
369
+ if not self._errored:
370
+ try:
371
+ await self.check_health(socket=socket)
372
+ except Exception as e:
373
+ self._errored = True
374
+ logger.exception(repr(e))
375
+
376
# NB: do before raising here so that the flag is set
377
# by the time the caller receives this exception
378
raise request_output
379
380
finished = request_output.finished
381
yield request_output
382
+
383
finally:
263
- if not finished:
384
+ # Request was canceled by the client.
385
+ if not finished and not self._errored:
386
await self.abort(request_id)
387
266
- async def check_health(self) -> None:
388
+ async def check_health(self,
389
+ socket: Optional[zmq.asyncio.Socket] = None
390
+ ) -> None:
391
"""Raise if unhealthy"""
392
269
- with self.socket() as socket:
270
-
271
- # Ping RPCServer with CHECK_HEALTH request.
272
- await socket.send(cloudpickle.dumps(RPCUtilityRequest.CHECK_HEALTH)
273
- )
274
-
275
- # Await the reply from the server.
276
- # TODO: do we need an internal timeout here?
277
- # Or do we expect the external probe to timeout and let this chill?
278
- health_message = cloudpickle.loads(await socket.recv())
279
-
280
- if isinstance(health_message, Exception):
281
- raise health_message
282
-
283
- if health_message != VLLM_RPC_HEALTHY_STR:
284
- raise ValueError("Expected healthy response from backend but got "
285
- "f{health_message}")
393
+ await self._send_one_way_rpc_request(
394
+ request=RPCUtilityRequest.IS_SERVER_HEALTHY,
395
+ error_message="Got Unhealthy response from RPC Server",
396
+ timeout=VLLM_RPC_HEALTH_TIMEOUT_MS,
397
+ socket=socket)
398
399
async def encode(self, *args,
400
**kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
vllm/entrypoints/openai/rpc/server.py
+50
-66
@@ -1,6 +1,6 @@
1
import asyncio
2
import signal
3
-from typing import Any, Coroutine
3
+from typing import Any, Coroutine, Union
4
5
import cloudpickle
6
import uvloop
@@ -9,14 +9,19 @@ import zmq.asyncio
9
from typing_extensions import Never
10
11
from vllm import AsyncEngineArgs, AsyncLLMEngine
12
-from vllm.entrypoints.openai.rpc import (VLLM_RPC_HEALTHY_STR,
13
- VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
12
+from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
13
+ ParallelConfig, SchedulerConfig)
14
+from vllm.entrypoints.openai.rpc import (VLLM_RPC_SUCCESS_STR,
15
+ VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
16
RPCGenerateRequest, RPCUtilityRequest)
17
from vllm.logger import init_logger
18
from vllm.usage.usage_lib import UsageContext
19
20
logger = init_logger(__name__)
21
22
+CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
23
+ SchedulerConfig, LoRAConfig]
24
+
25
26
class AsyncEngineRPCServer:
27
@@ -29,9 +34,10 @@ class AsyncEngineRPCServer:
34
# Initialize context.
35
self.context = zmq.asyncio.Context()
36
32
- # Init socket for readiness state.
33
- self.socket = self.context.socket(zmq.constants.ROUTER)
34
- self.socket.bind(rpc_path)
37
+ # Init socket.
38
+ self.socket = self.context.socket(zmq.constants.DEALER)
39
+ self.socket.set_hwm(VLLM_RPC_ZMQ_HWM)
40
+ self.socket.connect(rpc_path)
41
42
def cleanup(self):
43
"""Cleanup all resources."""
@@ -41,39 +47,27 @@ class AsyncEngineRPCServer:
47
# Clear the engine reference so that it can be GC'ed.
48
del self.engine
49
44
- async def get_model_config(self, identity):
45
- """Send the ModelConfig"""
46
- model_config = await self.engine.get_model_config()
47
-
48
- await self.socket.send_multipart(
49
- [identity, cloudpickle.dumps(model_config)])
50
-
51
- async def get_decoding_config(self, identity):
52
- """Send the DecodingConfig"""
53
- decoding_config = await self.engine.get_decoding_config()
54
-
55
- await self.socket.send_multipart(
56
- [identity, cloudpickle.dumps(decoding_config)])
57
-
58
- async def get_lora_config(self, identity):
59
- lora_config = await self.engine.get_lora_config()
60
-
61
- await self.socket.send_multipart(
62
- [identity, cloudpickle.dumps(lora_config)])
63
-
64
- async def get_scheduler_config(self, identity):
65
- """Send the SchedulerConfig"""
66
- parallel_config = await self.engine.get_scheduler_config()
67
-
68
- await self.socket.send_multipart(
69
- [identity, cloudpickle.dumps(parallel_config)])
50
+ async def get_config(self, identity, request):
51
+ try:
52
+ config: CONFIG_TYPE
53
+ if request == RPCUtilityRequest.GET_MODEL_CONFIG:
54
+ config = await self.engine.get_model_config()
55
+ elif request == RPCUtilityRequest.GET_DECODING_CONFIG:
56
+ config = await self.engine.get_decoding_config()
57
+ elif request == RPCUtilityRequest.GET_LORA_CONFIG:
58
+ config = await self.engine.get_lora_config()
59
+ elif request == RPCUtilityRequest.GET_SCHEDULER_CONFIG:
60
+ config = await self.engine.get_scheduler_config()
61
+ elif request == RPCUtilityRequest.GET_PARALLEL_CONFIG:
62
+ config = await self.engine.get_parallel_config()
63
+ else:
64
+ raise ValueError("Unknown Config Request: %s", request)
65
71
- async def get_parallel_config(self, identity):
72
- """Send the ParallelConfig"""
73
- parallel_config = await self.engine.get_parallel_config()
66
+ await self.socket.send_multipart(
67
+ [identity, cloudpickle.dumps(config)])
68
75
- await self.socket.send_multipart(
76
- [identity, cloudpickle.dumps(parallel_config)])
69
+ except Exception as e:
70
+ await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
71
72
async def is_tracing_enabled(self, identity):
73
"""Send the is_tracing_enabled flag"""
@@ -86,31 +80,23 @@ class AsyncEngineRPCServer:
80
"""Log stats and confirm success."""
81
await self.engine.do_log_stats()
82
89
- await self.socket.send_multipart([
90
- identity,
91
- cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
92
- ])
83
+ await self.socket.send_multipart(
84
+ [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
85
86
async def is_server_ready(self, identity):
87
"""Notify the client that we are ready."""
96
- await self.socket.send_multipart([
97
- identity,
98
- cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
99
- ])
88
+ await self.socket.send_multipart(
89
+ [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
90
91
async def abort(self, identity, request: RPCAbortRequest):
92
"""Abort request and notify the client of success."""
93
try:
94
# Abort the request in the llm engine.
95
await self.engine.abort(request.request_id)
106
- except Exception:
107
- logger.warning("Failed to abort request %s", request.request_id)
108
- finally:
109
- # Send confirmation to the client.
110
- await self.socket.send_multipart([
111
- identity,
112
- cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
113
- ])
96
+ result: Union[str, Exception] = VLLM_RPC_SUCCESS_STR
97
+ except Exception as e:
98
+ result = e
99
+ await self.socket.send_multipart([identity, cloudpickle.dumps(result)])
100
101
async def generate(self, identity, generate_request: RPCGenerateRequest):
102
try:
@@ -127,14 +113,14 @@ class AsyncEngineRPCServer:
113
[identity, cloudpickle.dumps(request_output)])
114
115
except Exception as e:
130
- ### Notify client of all failures
116
await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
117
118
async def check_health(self, identity):
119
try:
120
await self.engine.check_health()
121
await self.socket.send_multipart(
137
- [identity, cloudpickle.dumps(VLLM_RPC_HEALTHY_STR)])
122
+ [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
123
+
124
except Exception as e:
125
await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
126
@@ -151,21 +137,19 @@ class AsyncEngineRPCServer:
137
return self.abort(identity, request)
138
139
elif isinstance(request, RPCUtilityRequest):
154
- if request == RPCUtilityRequest.GET_MODEL_CONFIG:
155
- return self.get_model_config(identity)
156
- elif request == RPCUtilityRequest.GET_PARALLEL_CONFIG:
157
- return self.get_parallel_config(identity)
158
- elif request == RPCUtilityRequest.GET_DECODING_CONFIG:
159
- return self.get_decoding_config(identity)
160
- elif request == RPCUtilityRequest.GET_SCHEDULER_CONFIG:
161
- return self.get_scheduler_config(identity)
162
- elif request == RPCUtilityRequest.GET_LORA_CONFIG:
163
- return self.get_lora_config(identity)
140
+ if request in [
141
+ RPCUtilityRequest.GET_MODEL_CONFIG,
142
+ RPCUtilityRequest.GET_PARALLEL_CONFIG,
143
+ RPCUtilityRequest.GET_DECODING_CONFIG,
144
+ RPCUtilityRequest.GET_SCHEDULER_CONFIG,
145
+ RPCUtilityRequest.GET_LORA_CONFIG
146
+ ]:
147
+ return self.get_config(identity, request)
148
elif request == RPCUtilityRequest.DO_LOG_STATS:
149
return self.do_log_stats(identity)
150
elif request == RPCUtilityRequest.IS_SERVER_READY:
151
return self.is_server_ready(identity)
168
- elif request == RPCUtilityRequest.CHECK_HEALTH:
152
+ elif request == RPCUtilityRequest.IS_SERVER_HEALTHY:
153
return self.check_health(identity)
154
elif request == RPCUtilityRequest.IS_TRACING_ENABLED:
155
return self.is_tracing_enabled(identity)