A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Frontend] Minor optimizations to zmq decoupled front-end (#7957)
Co-authored-by: Robert Shaw <rshaw@neuralmagic>
Nick Hill committed
Aug 28, 2024 at 17:22 UTC
4289cad37f345873f49638d82d83087718841da5
2 files changed
+64
-65
vllm/entrypoints/openai/rpc/client.py
+37
-44
@@ -1,11 +1,13 @@
1
import asyncio
2
+import pickle
3
from contextlib import contextmanager, suppress
3
-from typing import Any, AsyncGenerator, Mapping, Optional
4
+from typing import Any, AsyncGenerator, Iterator, Mapping, Optional
5
from uuid import uuid4
6
7
import cloudpickle
8
import zmq
9
import zmq.asyncio
10
+from zmq.asyncio import Socket
11
12
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
13
ParallelConfig, SchedulerConfig)
@@ -115,18 +117,21 @@ class AsyncEngineRPCClient:
117
self.context.set(zmq.constants.MAX_SOCKETS, socket_limit)
118
119
# IPC connection to RPC Server (uses unix sockets).
118
- self.to_rpc_server = self.context.socket(zmq.constants.DEALER)
120
+ self.to_rpc_server: Socket = self.context.socket(zmq.constants.DEALER)
121
self.to_rpc_server.set_hwm(VLLM_RPC_ZMQ_HWM)
122
self.to_rpc_server.bind(rpc_path)
123
124
# In process proxy to RPC Server (uses memory-based messaging).
123
- self.from_api_server = self.context.socket(zmq.constants.ROUTER)
125
+ self.from_api_server: Socket = self.context.socket(
126
+ zmq.constants.ROUTER)
127
self.from_api_server.set_hwm(VLLM_RPC_ZMQ_HWM)
128
self.from_api_server.bind(INPROC_PROXY_PATH)
129
130
# Asyncio background task for the proxy.
128
- self.proxy_task = asyncio.create_task(
131
+ self.proxy_in_task = asyncio.create_task(
132
self.run_proxy(self.from_api_server, self.to_rpc_server))
133
+ self.proxy_out_task = asyncio.create_task(
134
+ self.run_proxy(self.to_rpc_server, self.from_api_server))
135
136
# Since we open 1 inproc socket per request, we have a hard cap on
137
# the number of requests that can run in vLLM w. frontend
@@ -136,20 +141,11 @@ class AsyncEngineRPCClient:
141
# 1 for generate(), 1 for abort(), do_log_stats(), check_health()
142
self.limit_concurrency = socket_limit // 2 - 2
143
139
- async def run_proxy(self, socket_from, socket_to):
144
+ async def run_proxy(self, socket_from: Socket, socket_to: Socket):
145
"""Background task that runs a proxy"""
141
- poller = zmq.asyncio.Poller()
142
- poller.register(socket_from, zmq.constants.POLLIN)
143
- poller.register(socket_to, zmq.constants.POLLIN)
146
while True:
145
- events_lst = await poller.poll()
146
- events = dict(events_lst)
147
- if socket_from in events:
148
- identity, msg = await socket_from.recv_multipart()
149
- await socket_to.send_multipart([identity, msg])
150
- if socket_to in events:
151
- identity, msg = await socket_to.recv_multipart()
152
- await socket_from.send_multipart([identity, msg])
147
+ frames = await socket_from.recv_multipart(copy=False)
148
+ await socket_to.send_multipart(frames, copy=False)
149
150
async def setup(self):
151
"""Setup the client before it starts sending server requests."""
@@ -180,7 +176,7 @@ class AsyncEngineRPCClient:
176
self.context.destroy()
177
178
@contextmanager
183
- def to_proxy_socket(self):
179
+ def to_proxy_socket(self) -> Iterator[Socket]:
180
# Connect to the RPCServer via the proxy.
181
182
# Raise a sensible error if the client was already closed.
@@ -208,7 +204,8 @@ class AsyncEngineRPCClient:
204
205
with self.to_proxy_socket() as socket:
206
# Ping RPCServer with a request.
211
- await socket.send_multipart([cloudpickle.dumps(request)])
207
+ await socket.send_multipart((cloudpickle.dumps(request), ),
208
+ copy=False)
209
210
# Make sure the server responds
211
if await socket.poll(timeout=self._data_timeout) == 0:
@@ -216,7 +213,8 @@ class AsyncEngineRPCClient:
213
f"{self._data_timeout} ms")
214
215
# Await the data from the Server.
219
- data = cloudpickle.loads(await socket.recv())
216
+ frame = await socket.recv(copy=False)
217
+ data = pickle.loads(frame.buffer)
218
219
if isinstance(data, Exception):
220
# Re-raise exceptions returned by the server
@@ -234,23 +232,22 @@ class AsyncEngineRPCClient:
232
233
return data
234
237
- async def _send_one_way_rpc_request(
238
- self,
239
- request: RPC_REQUEST_TYPE,
240
- error_message: str,
241
- socket: Optional[zmq.asyncio.Socket] = None):
235
+ async def _send_one_way_rpc_request(self,
236
+ request: RPC_REQUEST_TYPE,
237
+ error_message: str,
238
+ socket: Optional[Socket] = None):
239
"""Send one-way RPC request to trigger an action."""
240
244
- async def do_rpc_call(socket: zmq.asyncio.Socket,
245
- request: RPC_REQUEST_TYPE):
241
+ async def do_rpc_call(socket: Socket, request: RPC_REQUEST_TYPE):
242
247
- await socket.send_multipart([cloudpickle.dumps(request)])
243
+ await socket.send_multipart((cloudpickle.dumps(request), ))
244
245
if await socket.poll(timeout=self._data_timeout) == 0:
246
raise TimeoutError("Server didn't reply within "
247
f"{self._data_timeout} ms")
248
253
- return cloudpickle.loads(await socket.recv())
249
+ frame = await socket.recv(copy=False)
250
+ return pickle.loads(frame.buffer)
251
252
# Make a new socket connection.
253
if socket is None:
@@ -386,21 +383,19 @@ class AsyncEngineRPCClient:
383
try:
384
with self.to_proxy_socket() as socket:
385
# Send RPCGenerateRequest to the RPCServer.
389
- await socket.send_multipart([
390
- cloudpickle.dumps(
391
- RPCGenerateRequest(
392
- inputs=inputs,
393
- sampling_params=sampling_params,
394
- request_id=request_id,
395
- lora_request=lora_request,
396
- trace_headers=trace_headers,
397
- prompt_adapter_request=prompt_adapter_request))
398
- ])
386
+ await socket.send_multipart((cloudpickle.dumps(
387
+ RPCGenerateRequest(
388
+ inputs=inputs,
389
+ sampling_params=sampling_params,
390
+ request_id=request_id,
391
+ lora_request=lora_request,
392
+ trace_headers=trace_headers,
393
+ prompt_adapter_request=prompt_adapter_request)), ))
394
395
# Stream back the results from the RPC Server.
396
while not finished:
402
- message = await socket.recv()
403
- request_output = cloudpickle.loads(message)
397
+ message = await socket.recv(copy=False)
398
+ request_output = pickle.loads(message.buffer)
399
400
if isinstance(request_output, Exception):
401
# On exception, check if the server is still healthy
@@ -424,9 +419,7 @@ class AsyncEngineRPCClient:
419
if not finished and not self._errored:
420
await self.abort(request_id)
421
427
- async def check_health(self,
428
- socket: Optional[zmq.asyncio.Socket] = None
429
- ) -> None:
422
+ async def check_health(self, socket: Optional[Socket] = None) -> None:
423
"""Raise if unhealthy"""
424
425
await self._send_one_way_rpc_request(
@@ -451,4 +444,4 @@ class AsyncEngineRPCClient:
444
445
await self._send_one_way_rpc_request(
446
request=RPCUtilityRequest.STOP_PROFILE,
454
- error_message="RPCRequest STOP_PROFILE failed.")
\ No newline at end of file
447
+ error_message="RPCRequest STOP_PROFILE failed.")
vllm/entrypoints/openai/rpc/server.py
+27
-21
@@ -1,4 +1,5 @@
1
import asyncio
2
+import pickle
3
import signal
4
from typing import Any, Coroutine, Union
5
@@ -7,6 +8,8 @@ import uvloop
8
import zmq
9
import zmq.asyncio
10
from typing_extensions import Never
11
+from zmq import Frame # type: ignore[attr-defined]
12
+from zmq.asyncio import Socket
13
14
from vllm import AsyncEngineArgs, AsyncLLMEngine
15
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
@@ -35,7 +38,7 @@ class AsyncEngineRPCServer:
38
self.context = zmq.asyncio.Context()
39
40
# Init socket.
38
- self.socket = self.context.socket(zmq.constants.DEALER)
41
+ self.socket: Socket = self.context.socket(zmq.constants.DEALER)
42
self.socket.set_hwm(VLLM_RPC_ZMQ_HWM)
43
self.socket.connect(rpc_path)
44
@@ -63,30 +66,31 @@ class AsyncEngineRPCServer:
66
else:
67
raise ValueError("Unknown Config Request: %s", request)
68
66
- await self.socket.send_multipart(
67
- [identity, cloudpickle.dumps(config)])
69
+ await self.socket.send_multipart((identity, pickle.dumps(config)),
70
+ copy=False)
71
72
except Exception as e:
70
- await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
73
+ await self.socket.send_multipart((identity, pickle.dumps(e)),
74
+ copy=False)
75
76
async def is_tracing_enabled(self, identity):
77
"""Send the is_tracing_enabled flag"""
78
tracing_flag = await self.engine.is_tracing_enabled()
79
80
await self.socket.send_multipart(
77
- [identity, cloudpickle.dumps(tracing_flag)])
81
+ (identity, pickle.dumps(tracing_flag)))
82
83
async def do_log_stats(self, identity):
84
"""Log stats and confirm success."""
85
await self.engine.do_log_stats()
86
87
await self.socket.send_multipart(
84
- [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
88
+ (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
89
90
async def is_server_ready(self, identity):
91
"""Notify the client that we are ready."""
92
await self.socket.send_multipart(
89
- [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
93
+ (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
94
95
async def abort(self, identity, request: RPCAbortRequest):
96
"""Abort request and notify the client of success."""
@@ -96,7 +100,7 @@ class AsyncEngineRPCServer:
100
result: Union[str, Exception] = VLLM_RPC_SUCCESS_STR
101
except Exception as e:
102
result = e
99
- await self.socket.send_multipart([identity, cloudpickle.dumps(result)])
103
+ await self.socket.send_multipart((identity, pickle.dumps(result)))
104
105
async def generate(self, identity, generate_request: RPCGenerateRequest):
106
try:
@@ -110,45 +114,47 @@ class AsyncEngineRPCServer:
114
115
async for request_output in results_generator:
116
await self.socket.send_multipart(
113
- [identity, cloudpickle.dumps(request_output)])
117
+ (identity, pickle.dumps(request_output)), copy=False)
118
119
except Exception as e:
116
- await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
120
+ await self.socket.send_multipart((identity, pickle.dumps(e)),
121
+ copy=False)
122
123
async def check_health(self, identity):
124
try:
125
await self.engine.check_health()
126
await self.socket.send_multipart(
122
- [identity, cloudpickle.dumps(VLLM_RPC_SUCCESS_STR)])
127
+ (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
128
129
except Exception as e:
125
- await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
130
+ await self.socket.send_multipart((identity, pickle.dumps(e)),
131
+ copy=False)
132
133
async def start_profile(self, identity):
134
logger.info("Starting profiler...")
135
await self.engine.start_profile()
136
logger.info("Profiler started.")
137
132
- await self.socket.send_multipart([
138
+ await self.socket.send_multipart((
139
identity,
134
- cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
135
- ])
140
+ pickle.dumps(VLLM_RPC_SUCCESS_STR),
141
+ ))
142
143
async def stop_profile(self, identity):
144
logger.info("Stopping profiler...")
145
await self.engine.stop_profile()
146
logger.info("Profiler stopped.")
147
142
- await self.socket.send_multipart([
148
+ await self.socket.send_multipart((
149
identity,
144
- cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
145
- ])
150
+ pickle.dumps(VLLM_RPC_SUCCESS_STR),
151
+ ))
152
153
def _make_handler_coro(self, identity,
148
- message) -> Coroutine[Any, Any, Never]:
154
+ message: Frame) -> Coroutine[Any, Any, Never]:
155
"""Route the zmq message to the handler coroutine."""
156
151
- request = cloudpickle.loads(message)
157
+ request = cloudpickle.loads(message.buffer)
158
159
if isinstance(request, RPCGenerateRequest):
160
return self.generate(identity, request)
@@ -189,7 +195,7 @@ class AsyncEngineRPCServer:
195
running_tasks = set()
196
while True:
197
# Wait for a request.
192
- identity, message = await self.socket.recv_multipart()
198
+ identity, message = await self.socket.recv_multipart(copy=False)
199
200
# Process the request async.
201
task = asyncio.create_task(