@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 740374d456
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[core][distributed] fix zmq hang (#6759)

youkaichao committed Jul 24, 2024 at 17:37 UTC 740374d456a638df98ffbc7d9dab328752330e62
2 files changed +23 -41
vllm/connections.py
+2 -2
@@ -1,5 +1,5 @@
1 from pathlib import Path
2 -from typing import Mapping, Optional
2 +from typing import Mapping, MutableMapping, Optional
3 from urllib.parse import urlparse
4
5 import aiohttp
@@ -40,7 +40,7 @@ class HTTPConnection:
40 raise ValueError("Invalid HTTP URL: A valid HTTP URL "
41 "must have scheme 'http' or 'https'.")
42
43 - def _headers(self, **extras: str) -> Mapping[str, str]:
43 + def _headers(self, **extras: str) -> MutableMapping[str, str]:
44 return {"User-Agent": f"vLLM/{VLLM_VERSION}", **extras}
45
46 def get_response(
vllm/distributed/device_communicators/shm_broadcast.py
+21 -39
@@ -9,7 +9,7 @@ from unittest.mock import patch
9 import torch
10 import torch.distributed as dist
11 from torch.distributed import ProcessGroup
12 -from zmq import PUB, REP, REQ, SUB, SUBSCRIBE, Context # type: ignore
12 +from zmq import SUB, SUBSCRIBE, XPUB, XPUB_VERBOSE, Context # type: ignore
13
14 import vllm.envs as envs
15 from vllm.logger import init_logger
@@ -153,9 +153,7 @@ class Handle:
153
154 buffer: Optional[ShmRingBuffer] = None
155 local_subscribe_port: Optional[int] = None
156 - local_sync_port: Optional[int] = None
156 remote_subscribe_port: Optional[int] = None
158 - remote_sync_port: Optional[int] = None
157
158
159 class MessageQueue:
@@ -189,38 +187,36 @@ class MessageQueue:
187 self.buffer = ShmRingBuffer(n_local_reader, max_chunk_bytes,
188 max_chunks)
189
192 - self.local_socket = context.socket(PUB)
190 + # XPUB is very similar to PUB,
191 + # except that it can receive subscription messages
192 + # to confirm the number of subscribers
193 + self.local_socket = context.socket(XPUB)
194 + # set the verbose option so that we can receive every subscription
195 + # message. otherwise, we will only receive the first subscription
196 + # see http://api.zeromq.org/3-3:zmq-setsockopt for more details
197 + self.local_socket.setsockopt(XPUB_VERBOSE, True)
198 local_subscribe_port = get_open_port()
199 self.local_socket.bind(f"tcp://*:{local_subscribe_port}")
200
196 - self.local_sync_socket = context.socket(REP)
197 - local_sync_port = get_open_port()
198 - self.local_sync_socket.bind(f"tcp://*:{local_sync_port}")
201 self.current_idx = 0
202
203 else:
204 self.buffer = None # type: ignore
205 local_subscribe_port = None
204 - local_sync_port = None
206 self.local_socket = None
206 - self.local_sync_socket = None
207 self.current_idx = -1
208
209 if n_remote_reader > 0:
210 # for remote readers, we will:
211 # create a publish-subscribe socket to communicate large data
212 - self.remote_socket = context.socket(PUB)
212 + self.remote_socket = context.socket(XPUB)
213 + self.remote_socket.setsockopt(XPUB_VERBOSE, True)
214 remote_subscribe_port = get_open_port()
215 self.remote_socket.bind(f"tcp://*:{remote_subscribe_port}")
216
216 - self.remote_sync_socket = context.socket(REP)
217 - remote_sync_port = get_open_port()
218 - self.remote_sync_socket.bind(f"tcp://*:{remote_sync_port}")
217 else:
218 remote_subscribe_port = None
221 - remote_sync_port = None
219 self.remote_socket = None
223 - self.remote_sync_socket = None
220
221 self._is_writer = True
222 self._is_local_reader = False
@@ -233,9 +229,7 @@ class MessageQueue:
229 local_reader_ranks=local_reader_ranks,
230 buffer=self.buffer,
231 local_subscribe_port=local_subscribe_port,
236 - local_sync_port=local_sync_port,
232 remote_subscribe_port=remote_subscribe_port,
238 - remote_sync_port=remote_sync_port,
233 )
234
235 logger.info("vLLM message queue communication handle: %s", self.handle)
@@ -264,12 +258,7 @@ class MessageQueue:
258 self.local_socket.connect(
259 f"tcp://{handle.connect_ip}:{handle.local_subscribe_port}")
260
267 - self.local_sync_socket = context.socket(REQ)
268 - self.local_sync_socket.connect(
269 - f"tcp://{handle.connect_ip}:{handle.local_sync_port}")
270 -
261 self.remote_socket = None
272 - self.remote_sync_socket = None
262 else:
263 self.buffer = None # type: ignore
264 self.current_idx = -1
@@ -278,17 +267,12 @@ class MessageQueue:
267 self._is_remote_reader = True
268
269 self.local_socket = None
281 - self.local_sync_socket = None
270
271 self.remote_socket = context.socket(SUB)
272 self.remote_socket.setsockopt_string(SUBSCRIBE, "")
273 self.remote_socket.connect(
274 f"tcp://{handle.connect_ip}:{handle.remote_subscribe_port}")
275
288 - self.remote_sync_socket = context.socket(REQ)
289 - self.remote_sync_socket.connect(
290 - f"tcp://{handle.connect_ip}:{handle.remote_sync_port}")
291 -
276 return self
277
278 def wait_until_ready(self):
@@ -300,29 +284,27 @@ class MessageQueue:
284
285 # local readers
286 for i in range(self.n_local_reader):
303 - recv = self.local_sync_socket.recv()
304 - assert recv == b"READY"
305 - self.local_sync_socket.send(b"READY")
287 + # wait for subscription messages from all local readers
288 + self.local_socket.recv()
289 if self.n_local_reader > 0:
290 + # send a message to all local readers
291 + # to make sure the publish channel is working
292 self.local_socket.send(b"READY")
293
294 # remote readers
295 for i in range(self.n_remote_reader):
311 - recv = self.remote_sync_socket.recv()
312 - assert recv == b"READY"
313 - self.remote_sync_socket.send(b"READY")
296 + # wait for subscription messages from all remote readers
297 + self.remote_socket.recv()
298 if self.n_remote_reader > 0:
299 + # send a message to all remote readers
300 + # to make sure the publish channel is working
301 self.remote_socket.send(b"READY")
302 elif self._is_local_reader:
317 - self.local_sync_socket.send(b"READY")
318 - recv = self.local_sync_socket.recv()
319 - assert recv == b"READY"
303 + # wait for the writer to send a message
304 recv = self.local_socket.recv()
305 assert recv == b"READY"
306 elif self._is_remote_reader:
323 - self.remote_sync_socket.send(b"READY")
324 - recv = self.remote_sync_socket.recv()
325 - assert recv == b"READY"
307 + # wait for the writer to send a message
308 recv = self.remote_socket.recv()
309 assert recv == b"READY"
310