@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4289cad37f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Frontend] Minor optimizations to zmq decoupled front-end (#7957)

Co-authored-by: Robert Shaw <rshaw@neuralmagic>

Nick Hill committed Aug 28, 2024 at 17:22 UTC 4289cad37f345873f49638d82d83087718841da5
2 files changed +64 -65
vllm/entrypoints/openai/rpc/client.py
+37 -44
@@ -1,11 +1,13 @@
1 import asyncio
2 +import pickle
3 from contextlib import contextmanager, suppress
3 -from typing import Any, AsyncGenerator, Mapping, Optional
4 +from typing import Any, AsyncGenerator, Iterator, Mapping, Optional
5 from uuid import uuid4
6
7 import cloudpickle
8 import zmq
9 import zmq.asyncio
10 +from zmq.asyncio import Socket
11
12 from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
13 ParallelConfig, SchedulerConfig)
@@ -115,18 +117,21 @@ class AsyncEngineRPCClient:
117 self.context.set(zmq.constants.MAX_SOCKETS, socket_limit)
118
119 # IPC connection to RPC Server (uses unix sockets).
118 - self.to_rpc_server = self.context.socket(zmq.constants.DEALER)
120 + self.to_rpc_server: Socket = self.context.socket(zmq.constants.DEALER)
121 self.to_rpc_server.set_hwm(VLLM_RPC_ZMQ_HWM)
122 self.to_rpc_server.bind(rpc_path)
123
124 # In process proxy to RPC Server (uses memory-based messaging).
123 - self.from_api_server = self.context.socket(zmq.constants.ROUTER)
125 + self.from_api_server: Socket = self.context.socket(
126 + zmq.constants.ROUTER)
127 self.from_api_server.set_hwm(VLLM_RPC_ZMQ_HWM)
128 self.from_api_server.bind(INPROC_PROXY_PATH)
129
130 # Asyncio background task for the proxy.
128 - self.proxy_task = asyncio.create_task(
131 + self.proxy_in_task = asyncio.create_task(
132 self.run_proxy(self.from_api_server, self.to_rpc_server))
133 + self.proxy_out_task = asyncio.create_task(
134 + self.run_proxy(self.to_rpc_server, self.from_api_server))
135
136 # Since we open 1 inproc socket per request, we have a hard cap on
137 # the number of requests that can run in vLLM w. frontend
@@ -136,20 +141,11 @@ class AsyncEngineRPCClient:
141 # 1 for generate(), 1 for abort(), do_log_stats(), check_health()
142 self.limit_concurrency = socket_limit // 2 - 2
143
139 - async def run_proxy(self, socket_from, socket_to):
144 + async def run_proxy(self, socket_from: Socket, socket_to: Socket):
145 """Background task that runs a proxy"""
141 - poller = zmq.asyncio.Poller()
142 - poller.register(socket_from, zmq.constants.POLLIN)
143 - poller.register(socket_to, zmq.constants.POLLIN)
146 while True:
145 - events_lst = await poller.poll()
146 - events = dict(events_lst)
147 - if socket_from in events:
148 - identity, msg = await socket_from.recv_multipart()
149 - await socket_to.send_multipart([identity, msg])
150 - if socket_to in events:
151 - identity, msg = await socket_to.recv_multipart()
152 - await socket_from.send_multipart([identity, msg])
147 + frames = await socket_from.recv_multipart(copy=False)
148 + await socket_to.send_multipart(frames, copy=False)
149
150 async def setup(self):
151 """Setup the client before it starts sending server requests."""
@@ -180,7 +176,7 @@ class AsyncEngineRPCClient:
176 self.context.destroy()
177
178 @contextmanager
183 - def to_proxy_socket(self):
179 + def to_proxy_socket(self) -> Iterator[Socket]:
180 # Connect to the RPCServer via the proxy.
181
182 # Raise a sensible error if the client was already closed.
@@ -208,7 +204,8 @@ class AsyncEngineRPCClient:
204
205 with self.to_proxy_socket() as socket:
206 # Ping RPCServer with a request.
211 - await socket.send_multipart([cloudpickle.dumps(request)])
207 + await socket.send_multipart((cloudpickle.dumps(request), ),
208 + copy=False)
209
210 # Make sure the server responds
211 if await socket.poll(timeout=self._data_timeout) == 0:
@@ -216,7 +213,8 @@ class AsyncEngineRPCClient:
213 f"{self._data_timeout} ms")
214
215 # Await the data from the Server.
219 - data = cloudpickle.loads(await socket.recv())
216 + frame = await socket.recv(copy=False)
217 + data = pickle.loads(frame.buffer)
218
219 if isinstance(data, Exception):
220 # Re-raise exceptions returned by the server
@@ -234,23 +232,22 @@ class AsyncEngineRPCClient:
232
233 return data
234
237 - async def _send_one_way_rpc_request(
238 - self,
239 - request: RPC_REQUEST_TYPE,
240 - error_message: str,
241 - socket: Optional[zmq.asyncio.Socket] = None):
235 + async def _send_one_way_rpc_request(self,
236 + request: RPC_REQUEST_TYPE,
237 + error_message: str,
238 + socket: Optional[Socket] = None):
239 """Send one-way RPC request to trigger an action."""
240
244 - async def do_rpc_call(socket: zmq.asyncio.Socket,
245 - request: RPC_REQUEST_TYPE):
241 + async def do_rpc_call(socket: Socket, request: RPC_REQUEST_TYPE):
242
247 - await socket.send_multipart([cloudpickle.dumps(request)])
243 + await socket.send_multipart((cloudpickle.dumps(request), ))
244
245 if await socket.poll(timeout=self._data_timeout) == 0:
246 raise TimeoutError("Server didn't reply within "
247 f"{self._data_timeout} ms")
248
253 - return cloudpickle.loads(await socket.recv())
249 + frame = await socket.recv(copy=False)
250 + return pickle.loads(frame.buffer)
251
252 # Make a new socket connection.
253 if socket is None:
@@ -386,21 +383,19 @@ class AsyncEngineRPCClient:
383 try:
384 with self.to_proxy_socket() as socket:
385 # Send RPCGenerateRequest to the RPCServer.
389 - await socket.send_multipart([
390 - cloudpickle.dumps(
391 - RPCGenerateRequest(
392 - inputs=inputs,
393 - sampling_params=sampling_params,
394 - request_id=request_id,
395 - lora_request=lora_request,
396 - trace_headers=trace_headers,
397 - prompt_adapter_request=prompt_adapter_request))
398 - ])
386 + await socket.send_multipart((cloudpickle.dumps(
387 + RPCGenerateRequest(
388 + inputs=inputs,
389 + sampling_params=sampling_params,
390 + request_id=request_id,
391 + lora_request=lora_request,
392 + trace_headers=trace_headers,
393 + prompt_adapter_request=prompt_adapter_request)), ))
394
395 # Stream back the results from the RPC Server.
396 while not finished:
402 - message = await socket.recv()
403 - request_output = cloudpickle.loads(message)
397 + message = await socket.recv(copy=False)
398 + request_output = pickle.loads(message.buffer)
399
400 if isinstance(request_output, Exception):
401 # On exception, check if the server is still healthy
@@ -424,9 +419,7 @@ class AsyncEngineRPCClient:
419 if not finished and not self._errored:
420 await self.abort(request_id)
421
427 - async def check_health(self,
428 - socket: Optional[zmq.asyncio.Socket] = None
429 - ) -> None:
422 + async def check_health(self, socket: Optional[Socket] = None) -> None:
423 """Raise if unhealthy"""
424
425 await self._send_one_way_rpc_request(
@@ -451,4 +444,4 @@ class AsyncEngineRPCClient:
444
445 await self._send_one_way_rpc_request(
446 request=RPCUtilityRequest.STOP_PROFILE,
454 - error_message="RPCRequest STOP_PROFILE failed.")
\ No newline at end of file
447 + error_message="RPCRequest STOP_PROFILE failed.")
vllm/entrypoints/openai/rpc/server.py
+27 -21
@@ -1,4 +1,5 @@
1 import asyncio
2 +import pickle
3 import signal
4 from typing import Any, Coroutine, Union
5
@@ -7,6 +8,8 @@ import uvloop
8 import zmq
9 import zmq.asyncio
10 from typing_extensions import Never
11 +from zmq import Frame # type: ignore[attr-defined]
12 +from zmq.asyncio import Socket
13
14 from vllm import AsyncEngineArgs, AsyncLLMEngine
15 from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
@@ -35,7 +38,7 @@ class AsyncEngineRPCServer:
38 self.context = zmq.asyncio.Context()
39
40 # Init socket.
38 - self.socket = self.context.socket(zmq.constants.DEALER)
41 + self.socket: Socket = self.context.socket(zmq.constants.DEALER)
42 self.socket.set_hwm(VLLM_RPC_ZMQ_HWM)
43 self.socket.connect(rpc_path)
44
@@ -63,30 +66,31 @@ class AsyncEngineRPCServer:
66 else:
67 raise ValueError("Unknown Config Request: %s", request)
68
66 - await self.socket.send_multipart(
67 - [identity, cloudpickle.dumps(config)])
69 + await self.socket.send_multipart((identity, pickle.dumps(config)),
70 + copy=False)
71
72 except Exception as e:
70 - await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
73 + await self.socket.send_multipart((identity, pickle.dumps(e)),
74 + copy=False)
75
76 async def is_tracing_enabled(self, identity):
77 """Send the is_tracing_enabled flag"""
78 tracing_flag = await self.engine.is_tracing_enabled()
79
80 await self.socket.send_multipart(
77 - [identity, cloudpickle.dumps(tracing_flag)])
81 + (identity, pickle.dumps(tracing_flag)))
82
83 async def do_log_stats(self, identity):
84 """Log stats and confirm success."""
85 await self.engine.do_log_stats()
86
87 await self.socket.send_multipart(
84 - [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
88 + (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
89
90 async def is_server_ready(self, identity):
91 """Notify the client that we are ready."""
92 await self.socket.send_multipart(
89 - [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
93 + (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
94
95 async def abort(self, identity, request: RPCAbortRequest):
96 """Abort request and notify the client of success."""
@@ -96,7 +100,7 @@ class AsyncEngineRPCServer:
100 result: Union[str, Exception] = VLLM_RPC_SUCCESS_STR
101 except Exception as e:
102 result = e
99 - await self.socket.send_multipart([identity, cloudpickle.dumps(result)])
103 + await self.socket.send_multipart((identity, pickle.dumps(result)))
104
105 async def generate(self, identity, generate_request: RPCGenerateRequest):
106 try:
@@ -110,45 +114,47 @@ class AsyncEngineRPCServer:
114
115 async for request_output in results_generator:
116 await self.socket.send_multipart(
113 - [identity, cloudpickle.dumps(request_output)])
117 + (identity, pickle.dumps(request_output)), copy=False)
118
119 except Exception as e:
116 - await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
120 + await self.socket.send_multipart((identity, pickle.dumps(e)),
121 + copy=False)
122
123 async def check_health(self, identity):
124 try:
125 await self.engine.check_health()
126 await self.socket.send_multipart(
122 - [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
127 + (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
128
129 except Exception as e:
125 - await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
130 + await self.socket.send_multipart((identity, pickle.dumps(e)),
131 + copy=False)
132
133 async def start_profile(self, identity):
134 logger.info("Starting profiler...")
135 await self.engine.start_profile()
136 logger.info("Profiler started.")
137
132 - await self.socket.send_multipart([
138 + await self.socket.send_multipart((
139 identity,
134 - cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
135 - ])
140 + pickle.dumps(VLLM_RPC_SUCCESS_STR),
141 + ))
142
143 async def stop_profile(self, identity):
144 logger.info("Stopping profiler...")
145 await self.engine.stop_profile()
146 logger.info("Profiler stopped.")
147
142 - await self.socket.send_multipart([
148 + await self.socket.send_multipart((
149 identity,
144 - cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
145 - ])
150 + pickle.dumps(VLLM_RPC_SUCCESS_STR),
151 + ))
152
153 def _make_handler_coro(self, identity,
148 - message) -> Coroutine[Any, Any, Never]:
154 + message: Frame) -> Coroutine[Any, Any, Never]:
155 """Route the zmq message to the handler coroutine."""
156
151 - request = cloudpickle.loads(message)
157 + request = cloudpickle.loads(message.buffer)
158
159 if isinstance(request, RPCGenerateRequest):
160 return self.generate(identity, request)
@@ -189,7 +195,7 @@ class AsyncEngineRPCServer:
195 running_tasks = set()
196 while True:
197 # Wait for a request.
192 - identity, message = await self.socket.recv_multipart()
198 + identity, message = await self.socket.recv_multipart(copy=False)
199
200 # Process the request async.
201 task = asyncio.create_task(