@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f7e3b0c5aa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix][Frontend] Fix Issues Under High Load With `zeromq` Frontend (#7394)

Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Robert Shaw committed Aug 21, 2024 at 13:34 UTC f7e3b0c5aa2862d27b8872084c5ca934659ceef8
9 files changed +322 -141
.buildkite/test-pipeline.yaml
+1
@@ -86,6 +86,7 @@ steps:
86 - vllm/
87 commands:
88 - pip install -e ./plugins/vllm_add_dummy_model
89 + - pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@a4987bba6e9e9b3f22bd3a6c1ecf0abd04fd5622#egg=lm_eval[api]
90 - pytest -v -s entrypoints/llm
91 - pytest -v -s entrypoints/openai
92
tests/entrypoints/openai/test_accuracy.py new
+55
@@ -0,0 +1,55 @@
1 +"""
2 +This file test accuracy of the vLLM server via LMEval.
3 +It uses local-completions, which interacts with vLLM
4 +through the OAI API with N concurrent connections.
5 +This simulates real work usage of the API and makes
6 +sure that the zmq frontend mp RPC message passing and
7 +AsyncLLMEngine are working correctly.
8 +"""
9 +
10 +import lm_eval
11 +import pytest
12 +
13 +from ...utils import RemoteOpenAIServer
14 +
15 +MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
16 +NUM_CONCURRENT = 500
17 +TASK = "gsm8k"
18 +FILTER = "exact_match,strict-match"
19 +RTOL = 0.03
20 +EXPECTED_VALUE = 0.58
21 +
22 +
23 +@pytest.fixture(scope="module")
24 +def server():
25 + args = [
26 + "--max-model-len", "4096", "--enable-chunked-prefill",
27 + "--disable-log-requests", "--enforce-eager"
28 + ]
29 +
30 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
31 + yield remote_server
32 +
33 +
34 +@pytest.fixture(scope="module")
35 +def server_data(server):
36 + return {
37 + "url": f"{server.url_for('v1')}/completions",
38 + }
39 +
40 +
41 +def test_lm_eval_accuracy(server_data):
42 + model_args = (f"model={MODEL_NAME},"
43 + f"base_url={server_data['url']},"
44 + f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False")
45 +
46 + results = lm_eval.simple_evaluate(
47 + model="local-completions",
48 + model_args=model_args,
49 + tasks=TASK,
50 + )
51 +
52 + measured_value = results["results"][TASK][FILTER]
53 + assert (measured_value - RTOL < EXPECTED_VALUE
54 + and measured_value + RTOL > EXPECTED_VALUE
55 + ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
vllm/engine/async_llm_engine.py
+5
@@ -766,6 +766,11 @@ class AsyncLLMEngine:
766 def errored(self) -> bool:
767 return self._errored_with is not None
768
769 + @property
770 + def limit_concurrency(self) -> Optional[int]:
771 + """Maximum number of concurrently running requests."""
772 + return None
773 +
774 def set_errored(self, exc: Exception) -> None:
775 self._errored_with = exc
776
vllm/engine/protocol.py
+4
@@ -29,6 +29,10 @@ class AsyncEngineClient(Protocol):
29 def errored(self) -> bool:
30 ...
31
32 + @property
33 + def limit_concurrency(self) -> Optional[int]:
34 + """Maximum number of concurrently running requests."""
35 +
36 def generate(
37 self,
38 inputs: PromptInputs,
vllm/entrypoints/launcher.py
+9
@@ -27,6 +27,15 @@ async def serve_http(app: FastAPI, engine: AsyncEngineClient,
27
28 logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
29
30 + # Set concurrency limits in uvicorn if running in multiprocessing mode
31 + # since zmq has maximum socket limit of zmq.constants.SOCKET_LIMIT (65536).
32 + if engine.limit_concurrency is not None:
33 + logger.info(
34 + "Launching Uvicorn with --limit_concurrency %s. To avoid this "
35 + "limit at the expense of performance run with "
36 + "--disable-frontend-multiprocessing", engine.limit_concurrency)
37 + uvicorn_kwargs["limit_concurrency"] = engine.limit_concurrency
38 +
39 config = uvicorn.Config(app, **uvicorn_kwargs)
40 server = uvicorn.Server(config)
41 _add_shutdown_handlers(app, server, engine)
vllm/entrypoints/openai/api_server.py
+6 -5
@@ -135,6 +135,12 @@ async def build_async_engine_client(
135 logger.info("Multiprocessing frontend to use %s for RPC Path.",
136 rpc_path)
137
138 + # Build RPCClient, which conforms to AsyncEngineClient Protocol.
139 + # NOTE: Actually, this is not true yet. We still need to support
140 + # embedding models via RPC (see TODO above)
141 + rpc_client = AsyncEngineRPCClient(rpc_path)
142 + async_engine_client = rpc_client # type: ignore
143 +
144 # Start RPCServer in separate process (holds the AsyncLLMEngine).
145 context = multiprocessing.get_context("spawn")
146 # the current process might have CUDA context,
@@ -145,11 +151,6 @@ async def build_async_engine_client(
151 rpc_server_process.start()
152 logger.info("Started engine process with PID %d",
153 rpc_server_process.pid)
148 - # Build RPCClient, which conforms to AsyncEngineClient Protocol.
149 - # NOTE: Actually, this is not true yet. We still need to support
150 - # embedding models via RPC (see TODO above)
151 - rpc_client = AsyncEngineRPCClient(rpc_path)
152 - async_engine_client = rpc_client # type: ignore
154
155 try:
156 while True:
vllm/entrypoints/openai/rpc/__init__.py
+12 -2
@@ -7,8 +7,18 @@ from vllm.lora.request import LoRARequest
7 from vllm.prompt_adapter.request import PromptAdapterRequest
8 from vllm.sampling_params import SamplingParams
9
10 +# Success string used for RPC instructions.
11 VLLM_RPC_SUCCESS_STR = "SUCCESS"
11 -VLLM_RPC_HEALTHY_STR = "HEALTHY"
12 +
13 +# Timeouts.
14 +VLLM_RPC_SERVER_START_TIMEOUT_MS = 1000
15 +VLLM_RPC_HEALTH_TIMEOUT_MS = 10000
16 +
17 +# Minimum value of ZMQ.SOCKET_LIMIT to run mp.
18 +VLLM_RPC_SOCKET_LIMIT_CUTOFF = 2000
19 +
20 +# HWM is set to Infinity.
21 +VLLM_RPC_ZMQ_HWM = 0
22
23
24 @dataclass
@@ -34,7 +44,7 @@ class RPCUtilityRequest(Enum):
44 GET_SCHEDULER_CONFIG = 5
45 GET_LORA_CONFIG = 6
46 DO_LOG_STATS = 7
37 - CHECK_HEALTH = 8
47 + IS_SERVER_HEALTHY = 8
48 IS_TRACING_ENABLED = 9
49
50
vllm/entrypoints/openai/rpc/client.py
+180 -68
@@ -1,5 +1,7 @@
1 +import asyncio
2 from contextlib import contextmanager
3 from typing import Any, AsyncGenerator, Mapping, Optional
4 +from uuid import uuid4
5
6 import cloudpickle
7 import zmq
@@ -7,32 +9,140 @@ import zmq.asyncio
9
10 from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11 ParallelConfig, SchedulerConfig)
12 +# yapf: disable
13 from vllm.entrypoints.openai.rpc import (RPC_REQUEST_TYPE,
11 - VLLM_RPC_HEALTHY_STR,
12 - VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
14 + VLLM_RPC_HEALTH_TIMEOUT_MS,
15 + VLLM_RPC_SERVER_START_TIMEOUT_MS,
16 + VLLM_RPC_SOCKET_LIMIT_CUTOFF,
17 + VLLM_RPC_SUCCESS_STR,
18 + VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
19 RPCGenerateRequest, RPCUtilityRequest)
20 +# yapf: enable
21 from vllm.inputs import PromptInputs
22 +from vllm.logger import init_logger
23 from vllm.lora.request import LoRARequest
24 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
25 from vllm.prompt_adapter.request import PromptAdapterRequest
26 from vllm.sampling_params import SamplingParams
27 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
28
21 -# Time to wait before checking it the server process is alive.
22 -SERVER_START_TIMEOUT_MS = 1000
29 +logger = init_logger(__name__)
30 +
31 +# Path used for inprocess proxy.
32 +INPROC_PROXY_PATH = f"inproc://{uuid4()}"
33
34
35 class AsyncEngineRPCClient:
36 + """
37 + RPCClient that connects to the RPCServer wrapping AsyncLLMEngine.
38 +
39 + The overall design mirrors the Asynchronous Client Server Pattern
40 + https://zguide.zeromq.org/docs/chapter3/#The-Asynchronous-Client-Server-Pattern
41 +
42 + On startup, the RPCClient:
43 + - makes DEALER socket (to_rpc_server) that connects to the RPCServer
44 + via ipc, which uses unix sockets under the hood
45 + (https://libzmq.readthedocs.io/en/zeromq4-1/zmq_ipc.html)
46 + - makes ROUTER socket (from_api_server) that binds to a random
47 + inproc address, which uses memory under the hood
48 + (https://libzmq.readthedocs.io/en/zeromq3-x/zmq_inproc.html)
49 + - runs a proxy in a background asyncio task between
50 + from_api_server (ROUTER, inproc) and to_rpc_server (DEALER ipc, )
51 +
52 + Each request handled by the asyncio api_server calls generate():
53 + - make a DEALER socket that connects to from_api_server via inproc
54 + - send a RCPGenerateRequest to the inproc socket
55 + - background proxy forwards the request from inproc -> ipc
56 + - RPCServer responds to the request one token at a time over ipc
57 + - background proxy forwards the response from ipc -> inproc
58 +
59 + The connection looks like this:
60 + DEALER <- inproc -> [ ROUTER | DEALER ] <- ipc -> DEALER
61 +
62 + Message routing is performed via identities that are managed by the
63 + ROUTER socket. ROUTER sockets track every connection it has and
64 + tells the caller about these. The way it tells the caller is to stick
65 + the connection identity in front of each message received. When we
66 + send the message via a ROUTER, we first send an identity frame.
67 + See https://zguide.zeromq.org/docs/chapter3/#The-Extended-Reply-Envelope
68 + for more details on connection identities.
69 +
70 + This proxy design enables us to use a single unix socket, which
71 + improves performance by avoiding syscalls (~5%) and avoids resource limits
72 + such as ulimit, which defaults to 1024 on ubuntu.
73 +
74 + Note: we run set_hwm(0) on each socket, which sets the HWM to inf,
75 + which is required to avoid dropping messages under high load.
76 + This is generally not advisable. However, since we are in control
77 + of both sides of the connection + failure on either side is
78 + catastrophic to the overall system health and memory profiling
79 + suggests limited memory overhead relative to asyncio, we will
80 + proceed for now.
81 +
82 + See https://zguide.zeromq.org/docs/chapter2/#High-Water-Marks
83 + for more details on high water marks.
84 + """
85
86 def __init__(self, rpc_path: str):
87 self.context = zmq.asyncio.Context()
29 - self.rpc_path = rpc_path
88 +
89 + # Maximum number of sockets that can be opened (typically 65536).
90 + # ZMQ_SOCKET_LIMIT (http://api.zeromq.org/4-2:zmq-ctx-get)
91 + socket_limit = self.context.get(zmq.constants.SOCKET_LIMIT)
92 + if socket_limit < VLLM_RPC_SOCKET_LIMIT_CUTOFF:
93 + raise ValueError(
94 + f"Found zmq.constants.SOCKET_LIMIT={socket_limit}, which caps "
95 + "the number of concurrent requests vLLM can process. Launch "
96 + "vLLM with --disable-frontend-multiprocessing and open a "
97 + "GitHub issue so we can investigate.")
98 +
99 + # We only have 1 ipc connection that uses unix sockets, so
100 + # safe to set MAX_SOCKETS to the zmq SOCKET_LIMIT (i.e. will
101 + # not run into ulimit issues)
102 + self.context.set(zmq.constants.MAX_SOCKETS, socket_limit)
103 +
104 + # IPC connection to RPC Server (uses unix sockets).
105 + self.to_rpc_server = self.context.socket(zmq.constants.DEALER)
106 + self.to_rpc_server.set_hwm(VLLM_RPC_ZMQ_HWM)
107 + self.to_rpc_server.bind(rpc_path)
108 +
109 + # In process proxy to RPC Server (uses memory-based messaging).
110 + self.from_api_server = self.context.socket(zmq.constants.ROUTER)
111 + self.from_api_server.set_hwm(VLLM_RPC_ZMQ_HWM)
112 + self.from_api_server.bind(INPROC_PROXY_PATH)
113 +
114 + # Asyncio background task for the proxy.
115 + self.proxy_task = asyncio.create_task(
116 + self.run_proxy(self.from_api_server, self.to_rpc_server))
117 +
118 + # Since we open 1 inproc socket per request, we have a hard cap on
119 + # the number of requests that can run in vLLM w. frontend
120 + # mulitprocessing. This value is used uvicorn to launch
121 + # with --limit-concurrency to return 503 when server is overloaded.
122 + # We need 2 sockets per request - 2:
123 + # 1 for generate(), 1 for abort(), do_log_stats(), check_health()
124 + self.limit_concurrency = socket_limit // 2 - 2
125 +
126 + async def run_proxy(self, socket_from, socket_to):
127 + """Background task that runs a proxy"""
128 + poller = zmq.asyncio.Poller()
129 + poller.register(socket_from, zmq.constants.POLLIN)
130 + poller.register(socket_to, zmq.constants.POLLIN)
131 + while True:
132 + events = await poller.poll()
133 + events = dict(events)
134 + if socket_from in events:
135 + identity, msg = await socket_from.recv_multipart()
136 + await socket_to.send_multipart([identity, msg])
137 + if socket_to in events:
138 + identity, msg = await socket_to.recv_multipart()
139 + await socket_from.send_multipart([identity, msg])
140
141 async def setup(self):
142 """Setup the client before it starts sending server requests."""
143
144 # Wait until server is ready.
35 - await self.wait_for_server()
145 + await self._wait_for_server_rpc()
146 self._errored = False
147
148 # Get the configs.
@@ -51,29 +161,23 @@ class AsyncEngineRPCClient:
161
162 def close(self):
163 """Destroy the ZeroMQ Context."""
164 + # Close all sockets associated with this context and
165 + # then terminate the context.
166 + self.from_api_server.close()
167 + self.to_rpc_server.close()
168 self.context.destroy()
169
170 @contextmanager
57 - def socket(self):
58 - # Ensure client sockets are always closed after use
59 -
60 - # Connect to RPC socket for Request-Reply pattern,
171 + def to_proxy_socket(self):
172 + # Connect to the RPCServer via the proxy.
173 # Note that we use DEALER to enable asynchronous communication
174 # to enable streaming.
175 socket = self.context.socket(zmq.constants.DEALER)
176 + socket.set_hwm(VLLM_RPC_ZMQ_HWM)
177 try:
65 - socket.connect(self.rpc_path)
178 + socket.connect(INPROC_PROXY_PATH)
179 yield socket
180 finally:
68 - # linger == 0 means discard unsent messages
69 - # when the socket is closed. This is necessary
70 - # because otherwise self.context.destroy() will
71 - # wait for 30 seconds until unsent messages are
72 - # received, which is impossible if the server
73 - # crashed. In the absence of a server crash we
74 - # always expect a response before closing the
75 - # socket anyway.
76 - # Reference: http://api.zeromq.org/4-2:zmq-setsockopt#toc24
181 socket.close(linger=0)
182
183 async def _send_get_data_rpc_request(self, request: RPCUtilityRequest,
@@ -81,10 +185,9 @@ class AsyncEngineRPCClient:
185 error_message: str) -> Any:
186 """Send an RPC request that is expecting data back."""
187
84 - with self.socket() as socket:
85 -
188 + with self.to_proxy_socket() as socket:
189 # Ping RPCServer with a request.
87 - await socket.send(cloudpickle.dumps(request))
190 + await socket.send_multipart([cloudpickle.dumps(request)])
191
192 # Await the data from the Server.
193 data = cloudpickle.loads(await socket.recv())
@@ -93,31 +196,48 @@ class AsyncEngineRPCClient:
196 # LoRAConfig can be None.
197 if expected_type == LoRAConfig and data is None:
198 pass
199 + elif isinstance(data, Exception):
200 + logger.error(error_message)
201 + raise data
202 else:
203 raise ValueError(error_message)
204
205 return data
206
101 - async def _send_one_way_rpc_request(self,
102 - request: RPC_REQUEST_TYPE,
103 - error_message: str,
104 - timeout: Optional[int] = None):
207 + async def _send_one_way_rpc_request(
208 + self,
209 + request: RPC_REQUEST_TYPE,
210 + error_message: str,
211 + timeout: Optional[int] = None,
212 + socket: Optional[zmq.asyncio.Socket] = None):
213 """Send one-way RPC request to trigger an action."""
106 - with self.socket() as socket:
107 - # Ping RPC Server with request.
108 - await socket.send(cloudpickle.dumps(request))
214
110 - # Await acknowledgement from RPCServer.
215 + async def do_rpc_call(socket: zmq.asyncio.Socket,
216 + request: RPC_REQUEST_TYPE,
217 + timeout=None):
218 +
219 + await socket.send_multipart([cloudpickle.dumps(request)])
220 +
221 if timeout is not None and await socket.poll(timeout=timeout) == 0:
112 - raise TimeoutError(f"server didn't reply within {timeout} ms")
222 + raise TimeoutError(f"Server didn't reply within {timeout} ms")
223 +
224 + return cloudpickle.loads(await socket.recv())
225
114 - response = cloudpickle.loads(await socket.recv())
226 + # Make a new socket connection.
227 + if socket is None:
228 + with self.to_proxy_socket() as socket:
229 + response = await do_rpc_call(socket, request, timeout)
230 +
231 + # Use existing socket connection.
232 + else:
233 + response = await do_rpc_call(socket, request, timeout)
234
235 if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
236 + if isinstance(response, Exception):
237 + logger.error(error_message)
238 + raise response
239 raise ValueError(error_message)
240
119 - return response
120 -
241 async def get_tokenizer(self, lora_request: LoRARequest):
242 return await self.tokenizer.get_lora_tokenizer_async(lora_request)
243
@@ -130,13 +250,13 @@ class AsyncEngineRPCClient:
250 async def is_tracing_enabled(self) -> bool:
251 return self.tracing_flag
252
133 - async def wait_for_server(self):
253 + async def _wait_for_server_rpc(self):
254 """Wait for the RPCServer to start up."""
255
256 await self._send_one_way_rpc_request(
257 request=RPCUtilityRequest.IS_SERVER_READY,
138 - error_message="Unable to start RPC Server.",
139 - timeout=SERVER_START_TIMEOUT_MS)
258 + error_message="Unable to start RPC Server",
259 + timeout=VLLM_RPC_SERVER_START_TIMEOUT_MS)
260
261 async def _get_model_config_rpc(self) -> ModelConfig:
262 """Get the ModelConfig object from the RPC Server"""
@@ -184,8 +304,7 @@ class AsyncEngineRPCClient:
304 return await self._send_get_data_rpc_request(
305 RPCUtilityRequest.IS_TRACING_ENABLED,
306 expected_type=bool,
187 - error_message="Could not get is_tracing_enabled flag from RPC "
188 - "Server")
307 + error_message="Could not get is_tracing_enabled from RPC Server")
308
309 async def abort(self, request_id: str):
310 """Send an ABORT_REQUEST signal to the RPC Server"""
@@ -226,8 +345,7 @@ class AsyncEngineRPCClient:
345
346 finished = False
347 try:
229 - with self.socket() as socket:
230 -
348 + with self.to_proxy_socket() as socket:
349 # Send RPCGenerateRequest to the RPCServer.
350 await socket.send_multipart([
351 cloudpickle.dumps(
@@ -246,43 +364,37 @@ class AsyncEngineRPCClient:
364 request_output = cloudpickle.loads(message)
365
366 if isinstance(request_output, Exception):
249 - # On exception, check if the server is still healthy.
250 - # Use this to set the sync `is_running` and `errored`
251 - # properties.
252 - try:
253 - await self.check_health()
254 - except Exception:
255 - self._errored = True
367 + # On exception, check if the server is still healthy
368 + # possibly setting the `errored` property.
369 + if not self._errored:
370 + try:
371 + await self.check_health(socket=socket)
372 + except Exception as e:
373 + self._errored = True
374 + logger.exception(repr(e))
375 +
376 # NB: do before raising here so that the flag is set
377 # by the time the caller receives this exception
378 raise request_output
379
380 finished = request_output.finished
381 yield request_output
382 +
383 finally:
263 - if not finished:
384 + # Request was canceled by the client.
385 + if not finished and not self._errored:
386 await self.abort(request_id)
387
266 - async def check_health(self) -> None:
388 + async def check_health(self,
389 + socket: Optional[zmq.asyncio.Socket] = None
390 + ) -> None:
391 """Raise if unhealthy"""
392
269 - with self.socket() as socket:
270 -
271 - # Ping RPCServer with CHECK_HEALTH request.
272 - await socket.send(cloudpickle.dumps(RPCUtilityRequest.CHECK_HEALTH)
273 - )
274 -
275 - # Await the reply from the server.
276 - # TODO: do we need an internal timeout here?
277 - # Or do we expect the external probe to timeout and let this chill?
278 - health_message = cloudpickle.loads(await socket.recv())
279 -
280 - if isinstance(health_message, Exception):
281 - raise health_message
282 -
283 - if health_message != VLLM_RPC_HEALTHY_STR:
284 - raise ValueError("Expected healthy response from backend but got "
285 - "f{health_message}")
393 + await self._send_one_way_rpc_request(
394 + request=RPCUtilityRequest.IS_SERVER_HEALTHY,
395 + error_message="Got Unhealthy response from RPC Server",
396 + timeout=VLLM_RPC_HEALTH_TIMEOUT_MS,
397 + socket=socket)
398
399 async def encode(self, *args,
400 **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
vllm/entrypoints/openai/rpc/server.py
+50 -66
@@ -1,6 +1,6 @@
1 import asyncio
2 import signal
3 -from typing import Any, Coroutine
3 +from typing import Any, Coroutine, Union
4
5 import cloudpickle
6 import uvloop
@@ -9,14 +9,19 @@ import zmq.asyncio
9 from typing_extensions import Never
10
11 from vllm import AsyncEngineArgs, AsyncLLMEngine
12 -from vllm.entrypoints.openai.rpc import (VLLM_RPC_HEALTHY_STR,
13 - VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
12 +from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
13 + ParallelConfig, SchedulerConfig)
14 +from vllm.entrypoints.openai.rpc import (VLLM_RPC_SUCCESS_STR,
15 + VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
16 RPCGenerateRequest, RPCUtilityRequest)
17 from vllm.logger import init_logger
18 from vllm.usage.usage_lib import UsageContext
19
20 logger = init_logger(__name__)
21
22 +CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
23 + SchedulerConfig, LoRAConfig]
24 +
25
26 class AsyncEngineRPCServer:
27
@@ -29,9 +34,10 @@ class AsyncEngineRPCServer:
34 # Initialize context.
35 self.context = zmq.asyncio.Context()
36
32 - # Init socket for readiness state.
33 - self.socket = self.context.socket(zmq.constants.ROUTER)
34 - self.socket.bind(rpc_path)
37 + # Init socket.
38 + self.socket = self.context.socket(zmq.constants.DEALER)
39 + self.socket.set_hwm(VLLM_RPC_ZMQ_HWM)
40 + self.socket.connect(rpc_path)
41
42 def cleanup(self):
43 """Cleanup all resources."""
@@ -41,39 +47,27 @@ class AsyncEngineRPCServer:
47 # Clear the engine reference so that it can be GC'ed.
48 del self.engine
49
44 - async def get_model_config(self, identity):
45 - """Send the ModelConfig"""
46 - model_config = await self.engine.get_model_config()
47 -
48 - await self.socket.send_multipart(
49 - [identity, cloudpickle.dumps(model_config)])
50 -
51 - async def get_decoding_config(self, identity):
52 - """Send the DecodingConfig"""
53 - decoding_config = await self.engine.get_decoding_config()
54 -
55 - await self.socket.send_multipart(
56 - [identity, cloudpickle.dumps(decoding_config)])
57 -
58 - async def get_lora_config(self, identity):
59 - lora_config = await self.engine.get_lora_config()
60 -
61 - await self.socket.send_multipart(
62 - [identity, cloudpickle.dumps(lora_config)])
63 -
64 - async def get_scheduler_config(self, identity):
65 - """Send the SchedulerConfig"""
66 - parallel_config = await self.engine.get_scheduler_config()
67 -
68 - await self.socket.send_multipart(
69 - [identity, cloudpickle.dumps(parallel_config)])
50 + async def get_config(self, identity, request):
51 + try:
52 + config: CONFIG_TYPE
53 + if request == RPCUtilityRequest.GET_MODEL_CONFIG:
54 + config = await self.engine.get_model_config()
55 + elif request == RPCUtilityRequest.GET_DECODING_CONFIG:
56 + config = await self.engine.get_decoding_config()
57 + elif request == RPCUtilityRequest.GET_LORA_CONFIG:
58 + config = await self.engine.get_lora_config()
59 + elif request == RPCUtilityRequest.GET_SCHEDULER_CONFIG:
60 + config = await self.engine.get_scheduler_config()
61 + elif request == RPCUtilityRequest.GET_PARALLEL_CONFIG:
62 + config = await self.engine.get_parallel_config()
63 + else:
64 + raise ValueError("Unknown Config Request: %s", request)
65
71 - async def get_parallel_config(self, identity):
72 - """Send the ParallelConfig"""
73 - parallel_config = await self.engine.get_parallel_config()
66 + await self.socket.send_multipart(
67 + [identity, cloudpickle.dumps(config)])
68
75 - await self.socket.send_multipart(
76 - [identity, cloudpickle.dumps(parallel_config)])
69 + except Exception as e:
70 + await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
71
72 async def is_tracing_enabled(self, identity):
73 """Send the is_tracing_enabled flag"""
@@ -86,31 +80,23 @@ class AsyncEngineRPCServer:
80 """Log stats and confirm success."""
81 await self.engine.do_log_stats()
82
89 - await self.socket.send_multipart([
90 - identity,
91 - cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
92 - ])
83 + await self.socket.send_multipart(
84 + [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
85
86 async def is_server_ready(self, identity):
87 """Notify the client that we are ready."""
96 - await self.socket.send_multipart([
97 - identity,
98 - cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
99 - ])
88 + await self.socket.send_multipart(
89 + [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
90
91 async def abort(self, identity, request: RPCAbortRequest):
92 """Abort request and notify the client of success."""
93 try:
94 # Abort the request in the llm engine.
95 await self.engine.abort(request.request_id)
106 - except Exception:
107 - logger.warning("Failed to abort request %s", request.request_id)
108 - finally:
109 - # Send confirmation to the client.
110 - await self.socket.send_multipart([
111 - identity,
112 - cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
113 - ])
96 + result: Union[str, Exception] = VLLM_RPC_SUCCESS_STR
97 + except Exception as e:
98 + result = e
99 + await self.socket.send_multipart([identity, cloudpickle.dumps(result)])
100
101 async def generate(self, identity, generate_request: RPCGenerateRequest):
102 try:
@@ -127,14 +113,14 @@ class AsyncEngineRPCServer:
113 [identity, cloudpickle.dumps(request_output)])
114
115 except Exception as e:
130 - ### Notify client of all failures
116 await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
117
118 async def check_health(self, identity):
119 try:
120 await self.engine.check_health()
121 await self.socket.send_multipart(
137 - [identity, cloudpickle.dumps(VLLM_RPC_HEALTHY_STR)])
122 + [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
123 +
124 except Exception as e:
125 await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
126
@@ -151,21 +137,19 @@ class AsyncEngineRPCServer:
137 return self.abort(identity, request)
138
139 elif isinstance(request, RPCUtilityRequest):
154 - if request == RPCUtilityRequest.GET_MODEL_CONFIG:
155 - return self.get_model_config(identity)
156 - elif request == RPCUtilityRequest.GET_PARALLEL_CONFIG:
157 - return self.get_parallel_config(identity)
158 - elif request == RPCUtilityRequest.GET_DECODING_CONFIG:
159 - return self.get_decoding_config(identity)
160 - elif request == RPCUtilityRequest.GET_SCHEDULER_CONFIG:
161 - return self.get_scheduler_config(identity)
162 - elif request == RPCUtilityRequest.GET_LORA_CONFIG:
163 - return self.get_lora_config(identity)
140 + if request in [
141 + RPCUtilityRequest.GET_MODEL_CONFIG,
142 + RPCUtilityRequest.GET_PARALLEL_CONFIG,
143 + RPCUtilityRequest.GET_DECODING_CONFIG,
144 + RPCUtilityRequest.GET_SCHEDULER_CONFIG,
145 + RPCUtilityRequest.GET_LORA_CONFIG
146 + ]:
147 + return self.get_config(identity, request)
148 elif request == RPCUtilityRequest.DO_LOG_STATS:
149 return self.do_log_stats(identity)
150 elif request == RPCUtilityRequest.IS_SERVER_READY:
151 return self.is_server_ready(identity)
168 - elif request == RPCUtilityRequest.CHECK_HEALTH:
152 + elif request == RPCUtilityRequest.IS_SERVER_HEALTHY:
153 return self.check_health(identity)
154 elif request == RPCUtilityRequest.IS_TRACING_ENABLED:
155 return self.is_tracing_enabled(identity)