@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da78caecfa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[core][distributed] zmq fallback for broadcasting large objects (#6183)

[core][distributed] add zmq fallback for broadcasting large objects (#6183)

youkaichao committed Jul 9, 2024 at 18:49 UTC da78caecfa7f6137efc3e08388f4db102650ac45
6 files changed +274 -80
requirements-common.txt
+1
@@ -21,3 +21,4 @@ lm-format-enforcer == 0.10.1
21 outlines >= 0.0.43 # Requires torch >= 2.1.0
22 typing_extensions
23 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
24 +pyzmq
tests/distributed/test_same_node.py
+3 -2
@@ -2,10 +2,11 @@ import os
2
3 import torch
4
5 -from vllm.distributed.parallel_state import is_in_the_same_node
5 +from vllm.distributed.parallel_state import in_the_same_node_as
6
7 torch.distributed.init_process_group(backend="gloo")
8 -test_result = is_in_the_same_node(torch.distributed.group.WORLD)
8 +test_result = all(
9 + in_the_same_node_as(torch.distributed.group.WORLD, source_rank=0))
10
11 expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
12 assert test_result == expected, f"Expected {expected}, got {test_result}"
tests/distributed/test_shm_broadcast.py
+3 -14
@@ -6,8 +6,7 @@ from typing import List
6 import numpy as np
7 import torch.distributed as dist
8
9 -from vllm.distributed.device_communicators.shm_broadcast import (
10 - ShmRingBuffer, ShmRingBufferIO)
9 +from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
10 from vllm.utils import update_environment_variables
11
12
@@ -56,8 +55,8 @@ def worker_fn_wrapper(fn):
55 @worker_fn_wrapper
56 def worker_fn():
57 writer_rank = 2
59 - broadcaster = ShmRingBufferIO.create_from_process_group(
60 - dist.group.WORLD, 1024 * 1024, 2, writer_rank)
58 + broadcaster = MessageQueue.create_from_process_group(
59 + dist.group.WORLD, 40 * 1024, 2, writer_rank)
60 if dist.get_rank() == writer_rank:
61 seed = random.randint(0, 1000)
62 dist.broadcast_object_list([seed], writer_rank)
@@ -87,13 +86,3 @@ def worker_fn():
86
87 def test_shm_broadcast():
88 distributed_run(worker_fn, 4)
90 -
91 -
92 -def test_singe_process():
93 - buffer = ShmRingBuffer(1, 1024, 4)
94 - reader = ShmRingBufferIO(buffer, reader_rank=0)
95 - writer = ShmRingBufferIO(buffer, reader_rank=-1)
96 - writer.enqueue([0])
97 - writer.enqueue([1])
98 - assert reader.dequeue() == [0]
99 - assert reader.dequeue() == [1]
vllm/distributed/device_communicators/custom_all_reduce.py
+2 -2
@@ -9,7 +9,7 @@ import vllm.envs as envs
9 from vllm import _custom_ops as ops
10 from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11 gpu_p2p_access_check)
12 -from vllm.distributed.parallel_state import is_in_the_same_node
12 +from vllm.distributed.parallel_state import in_the_same_node_as
13 from vllm.logger import init_logger
14 from vllm.utils import cuda_device_count_stateless, is_full_nvlink
15
@@ -64,7 +64,7 @@ class CustomAllreduce:
64 assert dist.get_backend(group) != dist.Backend.NCCL, (
65 "CustomAllreduce should be attached to a non-NCCL group.")
66
67 - if not is_in_the_same_node(group):
67 + if not all(in_the_same_node_as(group, source_rank=0)):
68 # No need to initialize custom allreduce for multi-node case.
69 logger.warning(
70 "Custom allreduce is disabled because this process group"
vllm/distributed/device_communicators/shm_broadcast.py
+232 -37
@@ -1,16 +1,19 @@
1 import pickle
2 import time
3 from contextlib import contextmanager
4 +from dataclasses import dataclass, field
5 from multiprocessing import shared_memory
5 -from typing import Optional
6 +from typing import List, Optional
7 from unittest.mock import patch
8
9 import torch
10 import torch.distributed as dist
11 from torch.distributed import ProcessGroup
12 +from zmq import PUB, REP, REQ, SUB, SUBSCRIBE, Context # type: ignore
13
14 import vllm.envs as envs
15 from vllm.logger import init_logger
16 +from vllm.utils import get_ip, get_open_port
17
18 VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
19
@@ -135,18 +138,183 @@ class ShmRingBuffer:
138 yield buf
139
140
138 -class ShmRingBufferIO:
141 +@dataclass
142 +class Handle:
143 + connect_ip: str
144 + local_reader_ranks: List[int] = field(default_factory=list)
145
140 - def __init__(self, buffer: ShmRingBuffer, reader_rank: int):
141 - self.buffer = buffer
142 - self.reader_rank = reader_rank
143 - self._is_writer = self.reader_rank == -1
144 - self._is_reader = not self._is_writer
145 - if self._is_reader:
146 - assert 0 <= self.reader_rank < buffer.n_reader, \
147 - (f"Invalid reader rank {self.reader_rank} for buffer"
148 - f" created with {buffer.n_reader} readers")
149 - self.current_idx = 0
146 + buffer: Optional[ShmRingBuffer] = None
147 + local_subscribe_port: Optional[int] = None
148 + local_sync_port: Optional[int] = None
149 + remote_subscribe_port: Optional[int] = None
150 + remote_sync_port: Optional[int] = None
151 +
152 +
153 +class MessageQueue:
154 +
155 + def __init__(
156 + self,
157 + n_reader, # number of all readers
158 + n_local_reader, # number of local readers through shared memory
159 + local_reader_ranks: Optional[List[int]] = None,
160 + max_chunk_bytes: int = 1024 * 1024 * 10,
161 + max_chunks: int = 10,
162 + connect_ip: Optional[str] = None,
163 + ):
164 + if local_reader_ranks is None:
165 + local_reader_ranks = list(range(n_local_reader))
166 + else:
167 + assert len(local_reader_ranks) == n_local_reader
168 + self.n_local_reader = n_local_reader
169 + n_remote_reader = n_reader - n_local_reader
170 + self.n_remote_reader = n_remote_reader
171 +
172 + if connect_ip is None:
173 + connect_ip = get_ip()
174 +
175 + context = Context()
176 +
177 + if n_local_reader > 0:
178 + # for local readers, we will:
179 + # 1. create a shared memory ring buffer to communicate small data
180 + # 2. create a publish-subscribe socket to communicate large data
181 + self.buffer = ShmRingBuffer(n_local_reader, max_chunk_bytes,
182 + max_chunks)
183 +
184 + self.local_socket = context.socket(PUB)
185 + local_subscribe_port = get_open_port()
186 + self.local_socket.bind(f"tcp://*:{local_subscribe_port}")
187 +
188 + self.local_sync_socket = context.socket(REP)
189 + local_sync_port = get_open_port()
190 + self.local_sync_socket.bind(f"tcp://*:{local_sync_port}")
191 + self.current_idx = 0
192 +
193 + else:
194 + self.buffer = None # type: ignore
195 + local_subscribe_port = None
196 + local_sync_port = None
197 + self.local_socket = None
198 + self.local_sync_socket = None
199 + self.current_idx = -1
200 +
201 + if n_remote_reader > 0:
202 + # for remote readers, we will:
203 + # create a publish-subscribe socket to communicate large data
204 + self.remote_socket = context.socket(PUB)
205 + remote_subscribe_port = get_open_port()
206 + self.remote_socket.bind(f"tcp://*:{remote_subscribe_port}")
207 +
208 + self.remote_sync_socket = context.socket(REP)
209 + remote_sync_port = get_open_port()
210 + self.remote_sync_socket.bind(f"tcp://*:{remote_sync_port}")
211 + else:
212 + remote_subscribe_port = None
213 + remote_sync_port = None
214 + self.remote_socket = None
215 + self.remote_sync_socket = None
216 +
217 + self._is_writer = True
218 + self._is_local_reader = False
219 + self.local_reader_rank = -1
220 + # rank does not matter for remote readers
221 + self._is_remote_reader = False
222 +
223 + self.handle = Handle(
224 + connect_ip=connect_ip,
225 + local_reader_ranks=local_reader_ranks,
226 + buffer=self.buffer,
227 + local_subscribe_port=local_subscribe_port,
228 + local_sync_port=local_sync_port,
229 + remote_subscribe_port=remote_subscribe_port,
230 + remote_sync_port=remote_sync_port,
231 + )
232 +
233 + def export_handle(self) -> Handle:
234 + return self.handle
235 +
236 + @staticmethod
237 + def create_from_handle(handle: Handle, rank) -> "MessageQueue":
238 + self = MessageQueue.__new__(MessageQueue)
239 + self.handle = handle
240 + self._is_writer = False
241 +
242 + context = Context()
243 +
244 + if rank in handle.local_reader_ranks:
245 + assert handle.buffer is not None
246 + self.buffer = handle.buffer
247 + self.current_idx = 0
248 + self.local_reader_rank = handle.local_reader_ranks.index(rank)
249 + self._is_local_reader = True
250 + self._is_remote_reader = False
251 +
252 + self.local_socket = context.socket(SUB)
253 + self.local_socket.setsockopt_string(SUBSCRIBE, "")
254 + self.local_socket.connect(
255 + f"tcp://{handle.connect_ip}:{handle.local_subscribe_port}")
256 +
257 + self.local_sync_socket = context.socket(REQ)
258 + self.local_sync_socket.connect(
259 + f"tcp://{handle.connect_ip}:{handle.local_sync_port}")
260 +
261 + self.remote_socket = None
262 + self.remote_sync_socket = None
263 + else:
264 + self.buffer = None # type: ignore
265 + self.current_idx = -1
266 + self.local_reader_rank = -1
267 + self._is_local_reader = False
268 + self._is_remote_reader = True
269 +
270 + self.local_socket = None
271 + self.local_sync_socket = None
272 +
273 + self.remote_socket = context.socket(SUB)
274 + self.remote_socket.setsockopt_string(SUBSCRIBE, "")
275 + self.remote_socket.connect(
276 + f"tcp://{handle.connect_ip}:{handle.remote_subscribe_port}")
277 +
278 + self.remote_sync_socket = context.socket(REQ)
279 + self.remote_sync_socket.connect(
280 + f"tcp://{handle.connect_ip}:{handle.remote_sync_port}")
281 +
282 + return self
283 +
284 + def wait_until_ready(self):
285 + """This is a collective operation. All processes (including the
286 + readers and the writer) should call this function.
287 + """
288 + if self._is_writer:
289 + # wait for all readers to connect
290 +
291 + # local readers
292 + for i in range(self.n_local_reader):
293 + recv = self.local_sync_socket.recv()
294 + assert recv == b"READY"
295 + self.local_sync_socket.send(b"READY")
296 + if self.n_local_reader > 0:
297 + self.local_socket.send(b"READY")
298 +
299 + # remote readers
300 + for i in range(self.n_remote_reader):
301 + recv = self.remote_sync_socket.recv()
302 + assert recv == b"READY"
303 + self.remote_sync_socket.send(b"READY")
304 + if self.n_remote_reader > 0:
305 + self.remote_socket.send(b"READY")
306 + elif self._is_local_reader:
307 + self.local_sync_socket.send(b"READY")
308 + recv = self.local_sync_socket.recv()
309 + assert recv == b"READY"
310 + recv = self.local_socket.recv()
311 + assert recv == b"READY"
312 + elif self._is_remote_reader:
313 + self.remote_sync_socket.send(b"READY")
314 + recv = self.remote_sync_socket.recv()
315 + assert recv == b"READY"
316 + recv = self.remote_socket.recv()
317 + assert recv == b"READY"
318
319 @contextmanager
320 def acquire_write(self):
@@ -201,12 +369,12 @@ class ShmRingBufferIO:
369
370 @contextmanager
371 def acquire_read(self):
204 - assert self._is_reader, "Only readers can acquire read"
372 + assert self._is_local_reader, "Only readers can acquire read"
373 start_time = time.monotonic()
374 n_warning = 1
375 while True:
376 with self.buffer.get_metadata(self.current_idx) as metadata_buffer:
209 - read_flag = metadata_buffer[self.reader_rank + 1]
377 + read_flag = metadata_buffer[self.local_reader_rank + 1]
378 written_flag = metadata_buffer[0]
379 if not written_flag or read_flag:
380 # this block is either
@@ -236,7 +404,7 @@ class ShmRingBufferIO:
404
405 # caller has read from the buffer
406 # set the read flag
239 - metadata_buffer[self.reader_rank + 1] = 1
407 + metadata_buffer[self.local_reader_rank + 1] = 1
408 self.current_idx = (self.current_idx +
409 1) % self.buffer.max_chunks
410 break
@@ -244,21 +412,36 @@ class ShmRingBufferIO:
412 def enqueue(self, obj):
413 assert self._is_writer, "Only writers can enqueue"
414 serialized_obj = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
247 - if len(serialized_obj) > self.buffer.max_chunk_bytes:
248 - raise RuntimeError(
249 - f"{len(serialized_obj)=} larger than the allowed value "
250 - f"{self.buffer.max_chunk_bytes},"
251 - "Please increase the max_chunk_bytes parameter.")
252 - with self.acquire_write() as buf:
253 - buf[:len(serialized_obj)] = serialized_obj
415 + if self.n_local_reader > 0:
416 + if len(serialized_obj) >= self.buffer.max_chunk_bytes:
417 + with self.acquire_write() as buf:
418 + buf[0] = 1 # overflow
419 + self.local_socket.send(serialized_obj)
420 + else:
421 + with self.acquire_write() as buf:
422 + buf[0] = 0 # not overflow
423 + buf[1:len(serialized_obj) + 1] = serialized_obj
424 + if self.n_remote_reader > 0:
425 + self.remote_socket.send(serialized_obj)
426
427 def dequeue(self):
256 - assert self._is_reader, "Only readers can dequeue"
257 - with self.acquire_read() as buf:
258 - # no need to know the size of serialized object
259 - # pickle format itself contains the size information internally
260 - # see https://docs.python.org/3/library/pickle.html
261 - obj = pickle.loads(buf)
428 + if self._is_local_reader:
429 + overflow = False
430 + with self.acquire_read() as buf:
431 + overflow = buf[0] == 1
432 + if not overflow:
433 + # no need to know the size of serialized object
434 + # pickle format contains the size information internally
435 + # see https://docs.python.org/3/library/pickle.html
436 + obj = pickle.loads(buf[1:])
437 + if overflow:
438 + recv = self.local_socket.recv()
439 + obj = pickle.loads(recv)
440 + elif self._is_remote_reader:
441 + recv = self.remote_socket.recv()
442 + obj = pickle.loads(recv)
443 + else:
444 + raise RuntimeError("Only readers can dequeue")
445 return obj
446
447 def broadcast_object(self, obj=None):
@@ -272,24 +455,36 @@ class ShmRingBufferIO:
455 def create_from_process_group(pg: ProcessGroup,
456 max_chunk_bytes,
457 max_chunks,
275 - writer_rank=0) -> "ShmRingBufferIO":
458 + writer_rank=0) -> "MessageQueue":
459 group_rank = dist.get_rank(pg)
460 group_world_size = dist.get_world_size(pg)
278 - ranks_inside_group = list(range(group_world_size))
461 global_ranks = dist.get_process_group_ranks(pg)
462 +
463 + from vllm.distributed.parallel_state import in_the_same_node_as
464 + status = in_the_same_node_as(pg, source_rank=writer_rank)
465 + same_node_ranks = [i for i, s in enumerate(status) if s]
466 n_reader = group_world_size - 1
281 - buffer: ShmRingBuffer
467 + n_local_reader = len(same_node_ranks) - 1
468 + local_reader_ranks = [i for i in same_node_ranks if i != writer_rank]
469 + buffer_io: MessageQueue
470 if group_rank == writer_rank:
283 - buffer = ShmRingBuffer(n_reader, max_chunk_bytes, max_chunks)
284 - dist.broadcast_object_list([buffer],
471 + buffer_io = MessageQueue(
472 + n_reader=n_reader,
473 + n_local_reader=n_local_reader,
474 + local_reader_ranks=local_reader_ranks,
475 + max_chunk_bytes=max_chunk_bytes,
476 + max_chunks=max_chunks,
477 + )
478 + handle = buffer_io.export_handle()
479 + dist.broadcast_object_list([handle],
480 src=global_ranks[writer_rank],
481 group=pg)
287 - return ShmRingBufferIO(buffer, -1)
482 else:
483 recv = [None]
484 dist.broadcast_object_list(recv,
485 src=global_ranks[writer_rank],
486 group=pg)
293 - buffer = recv[0] # type: ignore
294 - rest_ranks = [r for r in ranks_inside_group if r != writer_rank]
295 - return ShmRingBufferIO(buffer, rest_ranks.index(group_rank))
487 + handle = recv[0] # type: ignore
488 + buffer_io = MessageQueue.create_from_handle(handle, group_rank)
489 + buffer_io.wait_until_ready()
490 + return buffer_io
vllm/distributed/parallel_state.py
+33 -25
@@ -124,7 +124,7 @@ class GroupCoordinator:
124 # communicators are only created for world size > 1
125 pynccl_comm: Optional[Any] # PyNccl communicator
126 ca_comm: Optional[Any] # Custom allreduce communicator
127 - shm_broadcaster: Optional[Any] # shared memory broadcaster
127 + mq_broadcaster: Optional[Any] # shared memory broadcaster
128
129 def __init__(
130 self,
@@ -133,6 +133,7 @@ class GroupCoordinator:
133 torch_distributed_backend: Union[str, Backend],
134 use_pynccl: bool,
135 use_custom_allreduce: bool,
136 + use_message_queue_broadcaster: bool = False,
137 ):
138
139 self.rank = torch.distributed.get_rank()
@@ -190,10 +191,10 @@ class GroupCoordinator:
191 self.ca_comm = None
192
193 from vllm.distributed.device_communicators.shm_broadcast import (
193 - ShmRingBufferIO)
194 - self.shm_broadcaster: Optional[ShmRingBufferIO] = None
195 - if self.world_size > 1 and is_in_the_same_node(self.cpu_group):
196 - self.shm_broadcaster = ShmRingBufferIO.create_from_process_group(
194 + MessageQueue)
195 + self.mq_broadcaster: Optional[MessageQueue] = None
196 + if use_message_queue_broadcaster and self.world_size > 1:
197 + self.mq_broadcaster = MessageQueue.create_from_process_group(
198 self.cpu_group, 1 << 22, 6)
199
200 @property
@@ -377,9 +378,9 @@ class GroupCoordinator:
378 # Bypass the function if we are using only 1 GPU.
379 if self.world_size == 1:
380 return obj
380 - if self.shm_broadcaster is not None:
381 - assert src == 0, "Shared memory broadcaster only supports src=0"
382 - return self.shm_broadcaster.broadcast_object(obj)
381 + if self.mq_broadcaster is not None:
382 + assert src == 0, "Message queue broadcaster only supports src=0"
383 + return self.mq_broadcaster.broadcast_object(obj)
384 if self.rank_in_group == src:
385 torch.distributed.broadcast_object_list([obj],
386 src=self.ranks[src],
@@ -696,8 +697,8 @@ class GroupCoordinator:
697 self.pynccl_comm = None
698 if self.ca_comm is not None:
699 self.ca_comm = None
699 - if self.shm_broadcaster is not None:
700 - self.shm_broadcaster = None
700 + if self.mq_broadcaster is not None:
701 + self.mq_broadcaster = None
702
703
704 _WORLD: Optional[GroupCoordinator] = None
@@ -720,10 +721,12 @@ def init_world_group(ranks: List[int], local_rank: int,
721
722
723 def init_model_parallel_group(
723 - group_ranks: List[List[int]],
724 - local_rank: int,
725 - backend: str,
726 - use_custom_allreduce: Optional[bool] = None) -> GroupCoordinator:
724 + group_ranks: List[List[int]],
725 + local_rank: int,
726 + backend: str,
727 + use_custom_allreduce: Optional[bool] = None,
728 + use_message_queue_broadcaster: bool = False,
729 +) -> GroupCoordinator:
730 if use_custom_allreduce is None:
731 use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
732 return GroupCoordinator(
@@ -732,6 +735,7 @@ def init_model_parallel_group(
735 torch_distributed_backend=backend,
736 use_pynccl=True,
737 use_custom_allreduce=use_custom_allreduce,
738 + use_message_queue_broadcaster=use_message_queue_broadcaster,
739 )
740
741
@@ -880,8 +884,12 @@ def initialize_model_parallel(
884 range(i * tensor_model_parallel_size,
885 (i + 1) * tensor_model_parallel_size))
886 group_ranks.append(ranks)
887 +
888 + # message queue broadcaster is only used in tensor model parallel group
889 _TP = init_model_parallel_group(group_ranks,
884 - get_world_group().local_rank, backend)
890 + get_world_group().local_rank,
891 + backend,
892 + use_message_queue_broadcaster=True)
893
894 # Build the pipeline model-parallel groups.
895 num_pipeline_model_parallel_groups: int = (world_size //
@@ -993,15 +1001,15 @@ def destroy_distributed_environment():
1001 torch.distributed.destroy_process_group()
1002
1003
996 -def is_in_the_same_node(pg: ProcessGroup):
1004 +def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1005 """
998 - This is a collective operation that checks if all processes in the group
999 - are in the same node. It tests if all processes are attached to the same
1006 + This is a collective operation that returns if each rank is in the same node
1007 + as the source rank. It tests if processes are attached to the same
1008 memory system (shared access to shared memory).
1009 """
1010 assert torch.distributed.get_backend(
1011 pg) != torch.distributed.Backend.NCCL, (
1004 - "is_in_the_same_node should be tested with a non-NCCL group.")
1012 + "in_the_same_node_as should be tested with a non-NCCL group.")
1013 # local rank inside the group
1014 rank = torch.distributed.get_rank(group=pg)
1015 world_size = torch.distributed.get_world_size(group=pg)
@@ -1017,19 +1025,19 @@ def is_in_the_same_node(pg: ProcessGroup):
1025
1026 try:
1027 with contextlib.suppress(OSError):
1020 - if rank == 0:
1028 + if rank == source_rank:
1029 # create a shared memory segment
1030 shm = shared_memory.SharedMemory(create=True, size=128)
1031 shm.buf[:len(magic_message)] = magic_message
1032 torch.distributed.broadcast_object_list([shm.name],
1025 - src=ranks[0],
1033 + src=ranks[source_rank],
1034 group=pg)
1027 - is_in_the_same_node[0] = 1
1035 + is_in_the_same_node[rank] = 1
1036 else:
1037 # try to open the shared memory segment
1038 recv = [None]
1039 torch.distributed.broadcast_object_list(recv,
1032 - src=ranks[0],
1040 + src=ranks[source_rank],
1041 group=pg)
1042 name = recv[0]
1043 # fix to https://stackoverflow.com/q/62748654/9191338
@@ -1050,8 +1058,8 @@ def is_in_the_same_node(pg: ProcessGroup):
1058
1059 # clean up the shared memory segment
1060 with contextlib.suppress(OSError):
1053 - if rank == 0 and shm:
1061 + if rank == source_rank and shm:
1062 shm.unlink()
1063 torch.distributed.all_reduce(is_in_the_same_node, group=pg)
1064
1057 - return is_in_the_same_node.sum().item() == world_size
1065 + return [x == 1 for x in is_in_the_same_node.tolist()]