A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[core][distributed] fix zmq hang (#6759)
youkaichao committed
Jul 24, 2024 at 17:37 UTC
740374d456a638df98ffbc7d9dab328752330e62
2 files changed
+23
-41
vllm/connections.py
+2
-2
@@ -1,5 +1,5 @@
1
from pathlib import Path
2
-from typing import Mapping, Optional
2
+from typing import Mapping, MutableMapping, Optional
3
from urllib.parse import urlparse
4
5
import aiohttp
@@ -40,7 +40,7 @@ class HTTPConnection:
40
raise ValueError("Invalid HTTP URL: A valid HTTP URL "
41
"must have scheme 'http' or 'https'.")
42
43
- def _headers(self, **extras: str) -> Mapping[str, str]:
43
+ def _headers(self, **extras: str) -> MutableMapping[str, str]:
44
return {"User-Agent": f"vLLM/{VLLM_VERSION}", **extras}
45
46
def get_response(
vllm/distributed/device_communicators/shm_broadcast.py
+21
-39
@@ -9,7 +9,7 @@ from unittest.mock import patch
9
import torch
10
import torch.distributed as dist
11
from torch.distributed import ProcessGroup
12
-from zmq import PUB, REP, REQ, SUB, SUBSCRIBE, Context # type: ignore
12
+from zmq import SUB, SUBSCRIBE, XPUB, XPUB_VERBOSE, Context # type: ignore
13
14
import vllm.envs as envs
15
from vllm.logger import init_logger
@@ -153,9 +153,7 @@ class Handle:
153
154
buffer: Optional[ShmRingBuffer] = None
155
local_subscribe_port: Optional[int] = None
156
- local_sync_port: Optional[int] = None
156
remote_subscribe_port: Optional[int] = None
158
- remote_sync_port: Optional[int] = None
157
158
159
class MessageQueue:
@@ -189,38 +187,36 @@ class MessageQueue:
187
self.buffer = ShmRingBuffer(n_local_reader, max_chunk_bytes,
188
max_chunks)
189
192
- self.local_socket = context.socket(PUB)
190
+ # XPUB is very similar to PUB,
191
+ # except that it can receive subscription messages
192
+ # to confirm the number of subscribers
193
+ self.local_socket = context.socket(XPUB)
194
+ # set the verbose option so that we can receive every subscription
195
+ # message. otherwise, we will only receive the first subscription
196
+ # see http://api.zeromq.org/3-3:zmq-setsockopt for more details
197
+ self.local_socket.setsockopt(XPUB_VERBOSE, True)
198
local_subscribe_port = get_open_port()
199
self.local_socket.bind(f"tcp://*:{local_subscribe_port}")
200
196
- self.local_sync_socket = context.socket(REP)
197
- local_sync_port = get_open_port()
198
- self.local_sync_socket.bind(f"tcp://*:{local_sync_port}")
201
self.current_idx = 0
202
203
else:
204
self.buffer = None # type: ignore
205
local_subscribe_port = None
204
- local_sync_port = None
206
self.local_socket = None
206
- self.local_sync_socket = None
207
self.current_idx = -1
208
209
if n_remote_reader > 0:
210
# for remote readers, we will:
211
# create a publish-subscribe socket to communicate large data
212
- self.remote_socket = context.socket(PUB)
212
+ self.remote_socket = context.socket(XPUB)
213
+ self.remote_socket.setsockopt(XPUB_VERBOSE, True)
214
remote_subscribe_port = get_open_port()
215
self.remote_socket.bind(f"tcp://*:{remote_subscribe_port}")
216
216
- self.remote_sync_socket = context.socket(REP)
217
- remote_sync_port = get_open_port()
218
- self.remote_sync_socket.bind(f"tcp://*:{remote_sync_port}")
217
else:
218
remote_subscribe_port = None
221
- remote_sync_port = None
219
self.remote_socket = None
223
- self.remote_sync_socket = None
220
221
self._is_writer = True
222
self._is_local_reader = False
@@ -233,9 +229,7 @@ class MessageQueue:
229
local_reader_ranks=local_reader_ranks,
230
buffer=self.buffer,
231
local_subscribe_port=local_subscribe_port,
236
- local_sync_port=local_sync_port,
232
remote_subscribe_port=remote_subscribe_port,
238
- remote_sync_port=remote_sync_port,
233
)
234
235
logger.info("vLLM message queue communication handle: %s", self.handle)
@@ -264,12 +258,7 @@ class MessageQueue:
258
self.local_socket.connect(
259
f"tcp://{handle.connect_ip}:{handle.local_subscribe_port}")
260
267
- self.local_sync_socket = context.socket(REQ)
268
- self.local_sync_socket.connect(
269
- f"tcp://{handle.connect_ip}:{handle.local_sync_port}")
270
-
261
self.remote_socket = None
272
- self.remote_sync_socket = None
262
else:
263
self.buffer = None # type: ignore
264
self.current_idx = -1
@@ -278,17 +267,12 @@ class MessageQueue:
267
self._is_remote_reader = True
268
269
self.local_socket = None
281
- self.local_sync_socket = None
270
271
self.remote_socket = context.socket(SUB)
272
self.remote_socket.setsockopt_string(SUBSCRIBE, "")
273
self.remote_socket.connect(
274
f"tcp://{handle.connect_ip}:{handle.remote_subscribe_port}")
275
288
- self.remote_sync_socket = context.socket(REQ)
289
- self.remote_sync_socket.connect(
290
- f"tcp://{handle.connect_ip}:{handle.remote_sync_port}")
291
-
276
return self
277
278
def wait_until_ready(self):
@@ -300,29 +284,27 @@ class MessageQueue:
284
285
# local readers
286
for i in range(self.n_local_reader):
303
- recv = self.local_sync_socket.recv()
304
- assert recv == b"READY"
305
- self.local_sync_socket.send(b"READY")
287
+ # wait for subscription messages from all local readers
288
+ self.local_socket.recv()
289
if self.n_local_reader > 0:
290
+ # send a message to all local readers
291
+ # to make sure the publish channel is working
292
self.local_socket.send(b"READY")
293
294
# remote readers
295
for i in range(self.n_remote_reader):
311
- recv = self.remote_sync_socket.recv()
312
- assert recv == b"READY"
313
- self.remote_sync_socket.send(b"READY")
296
+ # wait for subscription messages from all remote readers
297
+ self.remote_socket.recv()
298
if self.n_remote_reader > 0:
299
+ # send a message to all remote readers
300
+ # to make sure the publish channel is working
301
self.remote_socket.send(b"READY")
302
elif self._is_local_reader:
317
- self.local_sync_socket.send(b"READY")
318
- recv = self.local_sync_socket.recv()
319
- assert recv == b"READY"
303
+ # wait for the writer to send a message
304
recv = self.local_socket.recv()
305
assert recv == b"READY"
306
elif self._is_remote_reader:
323
- self.remote_sync_socket.send(b"READY")
324
- recv = self.remote_sync_socket.recv()
325
- assert recv == b"READY"
307
+ # wait for the writer to send a message
308
recv = self.remote_socket.recv()
309
assert recv == b"READY"
310