A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[core][distributed] zmq fallback for broadcasting large objects (#6183)
[core][distributed] add zmq fallback for broadcasting large objects (#6183)
youkaichao committed
Jul 9, 2024 at 18:49 UTC
da78caecfa7f6137efc3e08388f4db102650ac45
6 files changed
+274
-80
requirements-common.txt
+1
@@ -21,3 +21,4 @@ lm-format-enforcer == 0.10.1
21
outlines >= 0.0.43 # Requires torch >= 2.1.0
22
typing_extensions
23
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
24
+pyzmq
tests/distributed/test_same_node.py
+3
-2
@@ -2,10 +2,11 @@ import os
2
3
import torch
4
5
-from vllm.distributed.parallel_state import is_in_the_same_node
5
+from vllm.distributed.parallel_state import in_the_same_node_as
6
7
torch.distributed.init_process_group(backend="gloo")
8
-test_result = is_in_the_same_node(torch.distributed.group.WORLD)
8
+test_result = all(
9
+ in_the_same_node_as(torch.distributed.group.WORLD, source_rank=0))
10
11
expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
12
assert test_result == expected, f"Expected {expected}, got {test_result}"
tests/distributed/test_shm_broadcast.py
+3
-14
@@ -6,8 +6,7 @@ from typing import List
6
import numpy as np
7
import torch.distributed as dist
8
9
-from vllm.distributed.device_communicators.shm_broadcast import (
10
- ShmRingBuffer, ShmRingBufferIO)
9
+from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
10
from vllm.utils import update_environment_variables
11
12
@@ -56,8 +55,8 @@ def worker_fn_wrapper(fn):
55
@worker_fn_wrapper
56
def worker_fn():
57
writer_rank = 2
59
- broadcaster = ShmRingBufferIO.create_from_process_group(
60
- dist.group.WORLD, 1024 * 1024, 2, writer_rank)
58
+ broadcaster = MessageQueue.create_from_process_group(
59
+ dist.group.WORLD, 40 * 1024, 2, writer_rank)
60
if dist.get_rank() == writer_rank:
61
seed = random.randint(0, 1000)
62
dist.broadcast_object_list([seed], writer_rank)
@@ -87,13 +86,3 @@ def worker_fn():
86
87
def test_shm_broadcast():
88
distributed_run(worker_fn, 4)
90
-
91
-
92
-def test_singe_process():
93
- buffer = ShmRingBuffer(1, 1024, 4)
94
- reader = ShmRingBufferIO(buffer, reader_rank=0)
95
- writer = ShmRingBufferIO(buffer, reader_rank=-1)
96
- writer.enqueue([0])
97
- writer.enqueue([1])
98
- assert reader.dequeue() == [0]
99
- assert reader.dequeue() == [1]
vllm/distributed/device_communicators/custom_all_reduce.py
+2
-2
@@ -9,7 +9,7 @@ import vllm.envs as envs
9
from vllm import _custom_ops as ops
10
from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11
gpu_p2p_access_check)
12
-from vllm.distributed.parallel_state import is_in_the_same_node
12
+from vllm.distributed.parallel_state import in_the_same_node_as
13
from vllm.logger import init_logger
14
from vllm.utils import cuda_device_count_stateless, is_full_nvlink
15
@@ -64,7 +64,7 @@ class CustomAllreduce:
64
assert dist.get_backend(group) != dist.Backend.NCCL, (
65
"CustomAllreduce should be attached to a non-NCCL group.")
66
67
- if not is_in_the_same_node(group):
67
+ if not all(in_the_same_node_as(group, source_rank=0)):
68
# No need to initialize custom allreduce for multi-node case.
69
logger.warning(
70
"Custom allreduce is disabled because this process group"
vllm/distributed/device_communicators/shm_broadcast.py
+232
-37
@@ -1,16 +1,19 @@
1
import pickle
2
import time
3
from contextlib import contextmanager
4
+from dataclasses import dataclass, field
5
from multiprocessing import shared_memory
5
-from typing import Optional
6
+from typing import List, Optional
7
from unittest.mock import patch
8
9
import torch
10
import torch.distributed as dist
11
from torch.distributed import ProcessGroup
12
+from zmq import PUB, REP, REQ, SUB, SUBSCRIBE, Context # type: ignore
13
14
import vllm.envs as envs
15
from vllm.logger import init_logger
16
+from vllm.utils import get_ip, get_open_port
17
18
VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
19
@@ -135,18 +138,183 @@ class ShmRingBuffer:
138
yield buf
139
140
138
-class ShmRingBufferIO:
141
+@dataclass
142
+class Handle:
143
+ connect_ip: str
144
+ local_reader_ranks: List[int] = field(default_factory=list)
145
140
- def __init__(self, buffer: ShmRingBuffer, reader_rank: int):
141
- self.buffer = buffer
142
- self.reader_rank = reader_rank
143
- self._is_writer = self.reader_rank == -1
144
- self._is_reader = not self._is_writer
145
- if self._is_reader:
146
- assert 0 <= self.reader_rank < buffer.n_reader, \
147
- (f"Invalid reader rank {self.reader_rank} for buffer"
148
- f" created with {buffer.n_reader} readers")
149
- self.current_idx = 0
146
+ buffer: Optional[ShmRingBuffer] = None
147
+ local_subscribe_port: Optional[int] = None
148
+ local_sync_port: Optional[int] = None
149
+ remote_subscribe_port: Optional[int] = None
150
+ remote_sync_port: Optional[int] = None
151
+
152
+
153
+class MessageQueue:
154
+
155
+ def __init__(
156
+ self,
157
+ n_reader, # number of all readers
158
+ n_local_reader, # number of local readers through shared memory
159
+ local_reader_ranks: Optional[List[int]] = None,
160
+ max_chunk_bytes: int = 1024 * 1024 * 10,
161
+ max_chunks: int = 10,
162
+ connect_ip: Optional[str] = None,
163
+ ):
164
+ if local_reader_ranks is None:
165
+ local_reader_ranks = list(range(n_local_reader))
166
+ else:
167
+ assert len(local_reader_ranks) == n_local_reader
168
+ self.n_local_reader = n_local_reader
169
+ n_remote_reader = n_reader - n_local_reader
170
+ self.n_remote_reader = n_remote_reader
171
+
172
+ if connect_ip is None:
173
+ connect_ip = get_ip()
174
+
175
+ context = Context()
176
+
177
+ if n_local_reader > 0:
178
+ # for local readers, we will:
179
+ # 1. create a shared memory ring buffer to communicate small data
180
+ # 2. create a publish-subscribe socket to communicate large data
181
+ self.buffer = ShmRingBuffer(n_local_reader, max_chunk_bytes,
182
+ max_chunks)
183
+
184
+ self.local_socket = context.socket(PUB)
185
+ local_subscribe_port = get_open_port()
186
+ self.local_socket.bind(f"tcp://*:{local_subscribe_port}")
187
+
188
+ self.local_sync_socket = context.socket(REP)
189
+ local_sync_port = get_open_port()
190
+ self.local_sync_socket.bind(f"tcp://*:{local_sync_port}")
191
+ self.current_idx = 0
192
+
193
+ else:
194
+ self.buffer = None # type: ignore
195
+ local_subscribe_port = None
196
+ local_sync_port = None
197
+ self.local_socket = None
198
+ self.local_sync_socket = None
199
+ self.current_idx = -1
200
+
201
+ if n_remote_reader > 0:
202
+ # for remote readers, we will:
203
+ # create a publish-subscribe socket to communicate large data
204
+ self.remote_socket = context.socket(PUB)
205
+ remote_subscribe_port = get_open_port()
206
+ self.remote_socket.bind(f"tcp://*:{remote_subscribe_port}")
207
+
208
+ self.remote_sync_socket = context.socket(REP)
209
+ remote_sync_port = get_open_port()
210
+ self.remote_sync_socket.bind(f"tcp://*:{remote_sync_port}")
211
+ else:
212
+ remote_subscribe_port = None
213
+ remote_sync_port = None
214
+ self.remote_socket = None
215
+ self.remote_sync_socket = None
216
+
217
+ self._is_writer = True
218
+ self._is_local_reader = False
219
+ self.local_reader_rank = -1
220
+ # rank does not matter for remote readers
221
+ self._is_remote_reader = False
222
+
223
+ self.handle = Handle(
224
+ connect_ip=connect_ip,
225
+ local_reader_ranks=local_reader_ranks,
226
+ buffer=self.buffer,
227
+ local_subscribe_port=local_subscribe_port,
228
+ local_sync_port=local_sync_port,
229
+ remote_subscribe_port=remote_subscribe_port,
230
+ remote_sync_port=remote_sync_port,
231
+ )
232
+
233
+ def export_handle(self) -> Handle:
234
+ return self.handle
235
+
236
+ @staticmethod
237
+ def create_from_handle(handle: Handle, rank) -> "MessageQueue":
238
+ self = MessageQueue.__new__(MessageQueue)
239
+ self.handle = handle
240
+ self._is_writer = False
241
+
242
+ context = Context()
243
+
244
+ if rank in handle.local_reader_ranks:
245
+ assert handle.buffer is not None
246
+ self.buffer = handle.buffer
247
+ self.current_idx = 0
248
+ self.local_reader_rank = handle.local_reader_ranks.index(rank)
249
+ self._is_local_reader = True
250
+ self._is_remote_reader = False
251
+
252
+ self.local_socket = context.socket(SUB)
253
+ self.local_socket.setsockopt_string(SUBSCRIBE, "")
254
+ self.local_socket.connect(
255
+ f"tcp://{handle.connect_ip}:{handle.local_subscribe_port}")
256
+
257
+ self.local_sync_socket = context.socket(REQ)
258
+ self.local_sync_socket.connect(
259
+ f"tcp://{handle.connect_ip}:{handle.local_sync_port}")
260
+
261
+ self.remote_socket = None
262
+ self.remote_sync_socket = None
263
+ else:
264
+ self.buffer = None # type: ignore
265
+ self.current_idx = -1
266
+ self.local_reader_rank = -1
267
+ self._is_local_reader = False
268
+ self._is_remote_reader = True
269
+
270
+ self.local_socket = None
271
+ self.local_sync_socket = None
272
+
273
+ self.remote_socket = context.socket(SUB)
274
+ self.remote_socket.setsockopt_string(SUBSCRIBE, "")
275
+ self.remote_socket.connect(
276
+ f"tcp://{handle.connect_ip}:{handle.remote_subscribe_port}")
277
+
278
+ self.remote_sync_socket = context.socket(REQ)
279
+ self.remote_sync_socket.connect(
280
+ f"tcp://{handle.connect_ip}:{handle.remote_sync_port}")
281
+
282
+ return self
283
+
284
+ def wait_until_ready(self):
285
+ """This is a collective operation. All processes (including the
286
+ readers and the writer) should call this function.
287
+ """
288
+ if self._is_writer:
289
+ # wait for all readers to connect
290
+
291
+ # local readers
292
+ for i in range(self.n_local_reader):
293
+ recv = self.local_sync_socket.recv()
294
+ assert recv == b"READY"
295
+ self.local_sync_socket.send(b"READY")
296
+ if self.n_local_reader > 0:
297
+ self.local_socket.send(b"READY")
298
+
299
+ # remote readers
300
+ for i in range(self.n_remote_reader):
301
+ recv = self.remote_sync_socket.recv()
302
+ assert recv == b"READY"
303
+ self.remote_sync_socket.send(b"READY")
304
+ if self.n_remote_reader > 0:
305
+ self.remote_socket.send(b"READY")
306
+ elif self._is_local_reader:
307
+ self.local_sync_socket.send(b"READY")
308
+ recv = self.local_sync_socket.recv()
309
+ assert recv == b"READY"
310
+ recv = self.local_socket.recv()
311
+ assert recv == b"READY"
312
+ elif self._is_remote_reader:
313
+ self.remote_sync_socket.send(b"READY")
314
+ recv = self.remote_sync_socket.recv()
315
+ assert recv == b"READY"
316
+ recv = self.remote_socket.recv()
317
+ assert recv == b"READY"
318
319
@contextmanager
320
def acquire_write(self):
@@ -201,12 +369,12 @@ class ShmRingBufferIO:
369
370
@contextmanager
371
def acquire_read(self):
204
- assert self._is_reader, "Only readers can acquire read"
372
+ assert self._is_local_reader, "Only readers can acquire read"
373
start_time = time.monotonic()
374
n_warning = 1
375
while True:
376
with self.buffer.get_metadata(self.current_idx) as metadata_buffer:
209
- read_flag = metadata_buffer[self.reader_rank + 1]
377
+ read_flag = metadata_buffer[self.local_reader_rank + 1]
378
written_flag = metadata_buffer[0]
379
if not written_flag or read_flag:
380
# this block is either
@@ -236,7 +404,7 @@ class ShmRingBufferIO:
404
405
# caller has read from the buffer
406
# set the read flag
239
- metadata_buffer[self.reader_rank + 1] = 1
407
+ metadata_buffer[self.local_reader_rank + 1] = 1
408
self.current_idx = (self.current_idx +
409
1) % self.buffer.max_chunks
410
break
@@ -244,21 +412,36 @@ class ShmRingBufferIO:
412
def enqueue(self, obj):
413
assert self._is_writer, "Only writers can enqueue"
414
serialized_obj = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
247
- if len(serialized_obj) > self.buffer.max_chunk_bytes:
248
- raise RuntimeError(
249
- f"{len(serialized_obj)=} larger than the allowed value "
250
- f"{self.buffer.max_chunk_bytes},"
251
- "Please increase the max_chunk_bytes parameter.")
252
- with self.acquire_write() as buf:
253
- buf[:len(serialized_obj)] = serialized_obj
415
+ if self.n_local_reader > 0:
416
+ if len(serialized_obj) >= self.buffer.max_chunk_bytes:
417
+ with self.acquire_write() as buf:
418
+ buf[0] = 1 # overflow
419
+ self.local_socket.send(serialized_obj)
420
+ else:
421
+ with self.acquire_write() as buf:
422
+ buf[0] = 0 # not overflow
423
+ buf[1:len(serialized_obj) + 1] = serialized_obj
424
+ if self.n_remote_reader > 0:
425
+ self.remote_socket.send(serialized_obj)
426
427
def dequeue(self):
256
- assert self._is_reader, "Only readers can dequeue"
257
- with self.acquire_read() as buf:
258
- # no need to know the size of serialized object
259
- # pickle format itself contains the size information internally
260
- # see https://docs.python.org/3/library/pickle.html
261
- obj = pickle.loads(buf)
428
+ if self._is_local_reader:
429
+ overflow = False
430
+ with self.acquire_read() as buf:
431
+ overflow = buf[0] == 1
432
+ if not overflow:
433
+ # no need to know the size of serialized object
434
+ # pickle format contains the size information internally
435
+ # see https://docs.python.org/3/library/pickle.html
436
+ obj = pickle.loads(buf[1:])
437
+ if overflow:
438
+ recv = self.local_socket.recv()
439
+ obj = pickle.loads(recv)
440
+ elif self._is_remote_reader:
441
+ recv = self.remote_socket.recv()
442
+ obj = pickle.loads(recv)
443
+ else:
444
+ raise RuntimeError("Only readers can dequeue")
445
return obj
446
447
def broadcast_object(self, obj=None):
@@ -272,24 +455,36 @@ class ShmRingBufferIO:
455
def create_from_process_group(pg: ProcessGroup,
456
max_chunk_bytes,
457
max_chunks,
275
- writer_rank=0) -> "ShmRingBufferIO":
458
+ writer_rank=0) -> "MessageQueue":
459
group_rank = dist.get_rank(pg)
460
group_world_size = dist.get_world_size(pg)
278
- ranks_inside_group = list(range(group_world_size))
461
global_ranks = dist.get_process_group_ranks(pg)
462
+
463
+ from vllm.distributed.parallel_state import in_the_same_node_as
464
+ status = in_the_same_node_as(pg, source_rank=writer_rank)
465
+ same_node_ranks = [i for i, s in enumerate(status) if s]
466
n_reader = group_world_size - 1
281
- buffer: ShmRingBuffer
467
+ n_local_reader = len(same_node_ranks) - 1
468
+ local_reader_ranks = [i for i in same_node_ranks if i != writer_rank]
469
+ buffer_io: MessageQueue
470
if group_rank == writer_rank:
283
- buffer = ShmRingBuffer(n_reader, max_chunk_bytes, max_chunks)
284
- dist.broadcast_object_list([buffer],
471
+ buffer_io = MessageQueue(
472
+ n_reader=n_reader,
473
+ n_local_reader=n_local_reader,
474
+ local_reader_ranks=local_reader_ranks,
475
+ max_chunk_bytes=max_chunk_bytes,
476
+ max_chunks=max_chunks,
477
+ )
478
+ handle = buffer_io.export_handle()
479
+ dist.broadcast_object_list([handle],
480
src=global_ranks[writer_rank],
481
group=pg)
287
- return ShmRingBufferIO(buffer, -1)
482
else:
483
recv = [None]
484
dist.broadcast_object_list(recv,
485
src=global_ranks[writer_rank],
486
group=pg)
293
- buffer = recv[0] # type: ignore
294
- rest_ranks = [r for r in ranks_inside_group if r != writer_rank]
295
- return ShmRingBufferIO(buffer, rest_ranks.index(group_rank))
487
+ handle = recv[0] # type: ignore
488
+ buffer_io = MessageQueue.create_from_handle(handle, group_rank)
489
+ buffer_io.wait_until_ready()
490
+ return buffer_io
vllm/distributed/parallel_state.py
+33
-25
@@ -124,7 +124,7 @@ class GroupCoordinator:
124
# communicators are only created for world size > 1
125
pynccl_comm: Optional[Any] # PyNccl communicator
126
ca_comm: Optional[Any] # Custom allreduce communicator
127
- shm_broadcaster: Optional[Any] # shared memory broadcaster
127
+ mq_broadcaster: Optional[Any] # shared memory broadcaster
128
129
def __init__(
130
self,
@@ -133,6 +133,7 @@ class GroupCoordinator:
133
torch_distributed_backend: Union[str, Backend],
134
use_pynccl: bool,
135
use_custom_allreduce: bool,
136
+ use_message_queue_broadcaster: bool = False,
137
):
138
139
self.rank = torch.distributed.get_rank()
@@ -190,10 +191,10 @@ class GroupCoordinator:
191
self.ca_comm = None
192
193
from vllm.distributed.device_communicators.shm_broadcast import (
193
- ShmRingBufferIO)
194
- self.shm_broadcaster: Optional[ShmRingBufferIO] = None
195
- if self.world_size > 1 and is_in_the_same_node(self.cpu_group):
196
- self.shm_broadcaster = ShmRingBufferIO.create_from_process_group(
194
+ MessageQueue)
195
+ self.mq_broadcaster: Optional[MessageQueue] = None
196
+ if use_message_queue_broadcaster and self.world_size > 1:
197
+ self.mq_broadcaster = MessageQueue.create_from_process_group(
198
self.cpu_group, 1 << 22, 6)
199
200
@property
@@ -377,9 +378,9 @@ class GroupCoordinator:
378
# Bypass the function if we are using only 1 GPU.
379
if self.world_size == 1:
380
return obj
380
- if self.shm_broadcaster is not None:
381
- assert src == 0, "Shared memory broadcaster only supports src=0"
382
- return self.shm_broadcaster.broadcast_object(obj)
381
+ if self.mq_broadcaster is not None:
382
+ assert src == 0, "Message queue broadcaster only supports src=0"
383
+ return self.mq_broadcaster.broadcast_object(obj)
384
if self.rank_in_group == src:
385
torch.distributed.broadcast_object_list([obj],
386
src=self.ranks[src],
@@ -696,8 +697,8 @@ class GroupCoordinator:
697
self.pynccl_comm = None
698
if self.ca_comm is not None:
699
self.ca_comm = None
699
- if self.shm_broadcaster is not None:
700
- self.shm_broadcaster = None
700
+ if self.mq_broadcaster is not None:
701
+ self.mq_broadcaster = None
702
703
704
_WORLD: Optional[GroupCoordinator] = None
@@ -720,10 +721,12 @@ def init_world_group(ranks: List[int], local_rank: int,
721
722
723
def init_model_parallel_group(
723
- group_ranks: List[List[int]],
724
- local_rank: int,
725
- backend: str,
726
- use_custom_allreduce: Optional[bool] = None) -> GroupCoordinator:
724
+ group_ranks: List[List[int]],
725
+ local_rank: int,
726
+ backend: str,
727
+ use_custom_allreduce: Optional[bool] = None,
728
+ use_message_queue_broadcaster: bool = False,
729
+) -> GroupCoordinator:
730
if use_custom_allreduce is None:
731
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
732
return GroupCoordinator(
@@ -732,6 +735,7 @@ def init_model_parallel_group(
735
torch_distributed_backend=backend,
736
use_pynccl=True,
737
use_custom_allreduce=use_custom_allreduce,
738
+ use_message_queue_broadcaster=use_message_queue_broadcaster,
739
)
740
741
@@ -880,8 +884,12 @@ def initialize_model_parallel(
884
range(i * tensor_model_parallel_size,
885
(i + 1) * tensor_model_parallel_size))
886
group_ranks.append(ranks)
887
+
888
+ # message queue broadcaster is only used in tensor model parallel group
889
_TP = init_model_parallel_group(group_ranks,
884
- get_world_group().local_rank, backend)
890
+ get_world_group().local_rank,
891
+ backend,
892
+ use_message_queue_broadcaster=True)
893
894
# Build the pipeline model-parallel groups.
895
num_pipeline_model_parallel_groups: int = (world_size //
@@ -993,15 +1001,15 @@ def destroy_distributed_environment():
1001
torch.distributed.destroy_process_group()
1002
1003
996
-def is_in_the_same_node(pg: ProcessGroup):
1004
+def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1005
"""
998
- This is a collective operation that checks if all processes in the group
999
- are in the same node. It tests if all processes are attached to the same
1006
+ This is a collective operation that returns if each rank is in the same node
1007
+ as the source rank. It tests if processes are attached to the same
1008
memory system (shared access to shared memory).
1009
"""
1010
assert torch.distributed.get_backend(
1011
pg) != torch.distributed.Backend.NCCL, (
1004
- "is_in_the_same_node should be tested with a non-NCCL group.")
1012
+ "in_the_same_node_as should be tested with a non-NCCL group.")
1013
# local rank inside the group
1014
rank = torch.distributed.get_rank(group=pg)
1015
world_size = torch.distributed.get_world_size(group=pg)
@@ -1017,19 +1025,19 @@ def is_in_the_same_node(pg: ProcessGroup):
1025
1026
try:
1027
with contextlib.suppress(OSError):
1020
- if rank == 0:
1028
+ if rank == source_rank:
1029
# create a shared memory segment
1030
shm = shared_memory.SharedMemory(create=True, size=128)
1031
shm.buf[:len(magic_message)] = magic_message
1032
torch.distributed.broadcast_object_list([shm.name],
1025
- src=ranks[0],
1033
+ src=ranks[source_rank],
1034
group=pg)
1027
- is_in_the_same_node[0] = 1
1035
+ is_in_the_same_node[rank] = 1
1036
else:
1037
# try to open the shared memory segment
1038
recv = [None]
1039
torch.distributed.broadcast_object_list(recv,
1032
- src=ranks[0],
1040
+ src=ranks[source_rank],
1041
group=pg)
1042
name = recv[0]
1043
# fix to https://stackoverflow.com/q/62748654/9191338
@@ -1050,8 +1058,8 @@ def is_in_the_same_node(pg: ProcessGroup):
1058
1059
# clean up the shared memory segment
1060
with contextlib.suppress(OSError):
1053
- if rank == 0 and shm:
1061
+ if rank == source_rank and shm:
1062
shm.unlink()
1063
torch.distributed.all_reduce(is_in_the_same_node, group=pg)
1064
1057
- return is_in_the_same_node.sum().item() == world_size
1065
+ return [x == 1 for x in is_in_the_same_node.tolist()]