@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d9a252bc8e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Core][Distributed] add shm broadcast (#5399)

Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

youkaichao committed Jun 20, 2024 at 22:12 UTC d9a252bc8e8a2741d8a2997032a94208fb8f29d9
5 files changed +384 -10
.buildkite/test-pipeline.yaml
+3 -1
@@ -28,9 +28,11 @@ steps:
28
29 - label: Distributed Comm Ops Test
30 #mirror_hardwares: [amd]
31 - command: pytest -v -s distributed/test_comm_ops.py
31 working_dir: "/vllm-workspace/tests"
32 num_gpus: 2
33 + commands:
34 + - pytest -v -s distributed/test_comm_ops.py
35 + - pytest -v -s distributed/test_shm_broadcast.py
36
37 - label: Distributed Tests (2 GPUs)
38 mirror_hardwares: [amd]
tests/distributed/test_shm_broadcast.py new
+82
@@ -0,0 +1,82 @@
1 +import multiprocessing
2 +import random
3 +import time
4 +
5 +import torch.distributed as dist
6 +
7 +from vllm.distributed.device_communicators.shm_broadcast import (
8 + ShmRingBuffer, ShmRingBufferIO)
9 +from vllm.utils import update_environment_variables
10 +
11 +
12 +def distributed_run(fn, world_size):
13 + number_of_processes = world_size
14 + processes = []
15 + for i in range(number_of_processes):
16 + env = {}
17 + env['RANK'] = str(i)
18 + env['LOCAL_RANK'] = str(i)
19 + env['WORLD_SIZE'] = str(number_of_processes)
20 + env['LOCAL_WORLD_SIZE'] = str(number_of_processes)
21 + env['MASTER_ADDR'] = 'localhost'
22 + env['MASTER_PORT'] = '12345'
23 + p = multiprocessing.Process(target=fn, args=(env, ))
24 + processes.append(p)
25 + p.start()
26 +
27 + for p in processes:
28 + p.join()
29 +
30 + for p in processes:
31 + assert p.exitcode == 0
32 +
33 +
34 +def worker_fn_wrapper(fn):
35 + # `multiprocessing.Process` cannot accept environment variables directly
36 + # so we need to pass the environment variables as arguments
37 + # and update the environment variables in the function
38 + def wrapped_fn(env):
39 + update_environment_variables(env)
40 + dist.init_process_group(backend="gloo")
41 + fn()
42 +
43 + return wrapped_fn
44 +
45 +
46 +@worker_fn_wrapper
47 +def worker_fn():
48 + writer_rank = 2
49 + broadcaster = ShmRingBufferIO.create_from_process_group(
50 + dist.group.WORLD, 1024, 2, writer_rank)
51 + if dist.get_rank() == writer_rank:
52 + time.sleep(random.random())
53 + broadcaster.broadcast_object(0)
54 + time.sleep(random.random())
55 + broadcaster.broadcast_object({})
56 + time.sleep(random.random())
57 + broadcaster.broadcast_object([])
58 + else:
59 + time.sleep(random.random())
60 + a = broadcaster.broadcast_object(None)
61 + time.sleep(random.random())
62 + b = broadcaster.broadcast_object(None)
63 + time.sleep(random.random())
64 + c = broadcaster.broadcast_object(None)
65 + assert a == 0
66 + assert b == {}
67 + assert c == []
68 + dist.barrier()
69 +
70 +
71 +def test_shm_broadcast():
72 + distributed_run(worker_fn, 4)
73 +
74 +
75 +def test_singe_process():
76 + buffer = ShmRingBuffer(1, 1024, 4)
77 + reader = ShmRingBufferIO(buffer, reader_rank=0)
78 + writer = ShmRingBufferIO(buffer, reader_rank=-1)
79 + writer.enqueue([0])
80 + writer.enqueue([1])
81 + assert reader.dequeue() == [0]
82 + assert reader.dequeue() == [1]
vllm/distributed/device_communicators/shm_broadcast.py new
+259
@@ -0,0 +1,259 @@
1 +import pickle
2 +import time
3 +from contextlib import contextmanager
4 +from multiprocessing import shared_memory
5 +from typing import Optional
6 +from unittest.mock import patch
7 +
8 +import torch
9 +import torch.distributed as dist
10 +from torch.distributed import ProcessGroup
11 +
12 +import vllm.envs as envs
13 +from vllm.logger import init_logger
14 +
15 +VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
16 +
17 +logger = init_logger(__name__)
18 +
19 +
20 +class ShmRingBuffer:
21 +
22 + def __init__(self,
23 + n_reader: int,
24 + max_chunk_bytes: int,
25 + max_chunks: int,
26 + name: Optional[str] = None):
27 + """
28 + A shared memory ring buffer implementation for broadcast communication.
29 + Essentially, it is a queue where only one will `enqueue` and multiple
30 + will `dequeue`. The max size of each item, together with the max number
31 + of items that can be stored in the buffer are known in advance.
32 + In this case, we don't need to synchronize the access to
33 + the buffer.
34 +
35 + Buffer memory layout:
36 + data metadata
37 + | |
38 + | (current_idx) | (current_idx)
39 + v v
40 + +-------------------------------+----------------------------------------+
41 + | chunk0 | chunk1 | ... | chunk | metadata0 | metadata1 | ... | metadata |
42 + +-------------------------------+----------------------------------------+
43 + | max_chunks x max_chunk_bytes | max_chunks x (1 + n_reader) bytes |
44 +
45 + metadata memory layout: each byte is a flag, the first byte is the written
46 + flag, and the rest are reader flags. The flags are set to 0 by default.
47 + +--------------+--------------+--------------+-----+--------------+
48 + | written_flag | reader0_flag | reader1_flag | ... | readerN_flag |
49 + +--------------+--------------+--------------+-----+--------------+
50 +
51 + During creation, `name` is None and the buffer is created. We can pass the
52 + created object to other processes by pickling it. The other processes will
53 + get the name of the shared memory and open it, so that they can access the
54 + same shared memory buffer.
55 + """# noqa
56 + self.n_reader = n_reader
57 + self.metadata_size = 1 + n_reader
58 + self.max_chunk_bytes = max_chunk_bytes
59 + self.max_chunks = max_chunks
60 + self.total_bytes_of_buffer = (self.max_chunk_bytes +
61 + self.metadata_size) * self.max_chunks
62 + self.data_offset = 0
63 + self.metadata_offset = self.max_chunk_bytes * self.max_chunks
64 +
65 + if name is None:
66 + # we are creating a buffer
67 + self.is_creator = True
68 + self.shared_memory = shared_memory.SharedMemory(
69 + create=True, size=self.total_bytes_of_buffer)
70 + # initialize the metadata section to 0
71 + with memoryview(self.shared_memory.buf[self.metadata_offset:]
72 + ) as metadata_buffer:
73 + torch.frombuffer(metadata_buffer, dtype=torch.uint8).fill_(0)
74 + else:
75 + # we are opening an existing buffer
76 + self.is_creator = False
77 + # fix to https://stackoverflow.com/q/62748654/9191338
78 + # Python incorrectly tracks shared memory even if it is not
79 + # created by the process. The following patch is a workaround.
80 + with patch("multiprocessing.resource_tracker.register",
81 + lambda *args, **kwargs: None):
82 + self.shared_memory = shared_memory.SharedMemory(name=name)
83 + assert self.shared_memory.size == self.total_bytes_of_buffer
84 + with memoryview(self.shared_memory.buf[self.metadata_offset:]
85 + ) as metadata_buffer:
86 + tensor = torch.frombuffer(metadata_buffer, dtype=torch.uint8)
87 + assert torch.all(tensor == 0)
88 +
89 + def __reduce__(self):
90 + return (
91 + self.__class__,
92 + (self.n_reader, self.max_chunk_bytes, self.max_chunks,
93 + self.shared_memory.name),
94 + )
95 +
96 + def __del__(self):
97 + self.shared_memory.close()
98 + if self.is_creator:
99 + self.shared_memory.unlink()
100 +
101 + @contextmanager
102 + def get_data(self, current_idx: int):
103 + start = self.data_offset + current_idx * self.max_chunk_bytes
104 + end = start + self.max_chunk_bytes
105 + with memoryview(self.shared_memory.buf[start:end]) as buf:
106 + yield buf
107 +
108 + @contextmanager
109 + def get_metadata(self, current_idx: int):
110 + start = self.metadata_offset + current_idx * self.metadata_size
111 + end = start + self.metadata_size
112 + with memoryview(self.shared_memory.buf[start:end]) as buf:
113 + yield buf
114 +
115 +
116 +class ShmRingBufferIO:
117 +
118 + def __init__(self, buffer: ShmRingBuffer, reader_rank: int):
119 + self.buffer = buffer
120 + self.reader_rank = reader_rank
121 + self._is_writer = self.reader_rank == -1
122 + self._is_reader = not self._is_writer
123 + if self._is_reader:
124 + assert 0 <= self.reader_rank < buffer.n_reader, \
125 + (f"Invalid reader rank {self.reader_rank} for buffer"
126 + f" created with {buffer.n_reader} readers")
127 + self.current_idx = 0
128 +
129 + @contextmanager
130 + def acquire_write(self):
131 + assert self._is_writer, "Only writers can acquire write"
132 + start_index = self.current_idx
133 + start_time = time.time()
134 + n_warning = 1
135 + while True:
136 + with self.buffer.get_metadata(self.current_idx) as metadata_buffer:
137 + read_count = sum(metadata_buffer[1:])
138 + written_flag = metadata_buffer[0]
139 + if written_flag and read_count != self.buffer.n_reader:
140 + # this block is written and not read by all readers
141 + # try to write to the next block
142 + self.current_idx = (self.current_idx +
143 + 1) % self.buffer.max_chunks
144 + if self.current_idx == start_index:
145 + # no empty block found
146 + if time.time(
147 + ) - start_time > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning: # noqa
148 + logger.warning(
149 + "No available block found in %s second. ",
150 + VLLM_RINGBUFFER_WARNING_INTERVAL)
151 + n_warning += 1
152 + # wait for a while (0.1 us)
153 + time.sleep(1e-7)
154 + continue
155 + # found a block that is either
156 + # (1) not written
157 + # (2) read by all readers
158 +
159 + # mark the block as not written
160 + metadata_buffer[0] = 0
161 + # let caller write to the buffer
162 + with self.buffer.get_data(self.current_idx) as buf:
163 + yield buf
164 +
165 + # caller has written to the buffer
166 + # mark the block as written
167 + metadata_buffer[0] = 1
168 + for i in range(1, self.buffer.n_reader + 1):
169 + # set read flag to 0, meaning it is not read yet
170 + metadata_buffer[i] = 0
171 + break
172 +
173 + @contextmanager
174 + def acquire_read(self):
175 + assert self._is_reader, "Only readers can acquire read"
176 + start_index = self.current_idx
177 + start_time = time.time()
178 + n_warning = 1
179 + while True:
180 + with self.buffer.get_metadata(self.current_idx) as metadata_buffer:
181 + read_flag = metadata_buffer[self.reader_rank + 1]
182 + written_flag = metadata_buffer[0]
183 + if not written_flag or read_flag:
184 + # this block is either
185 + # (1) not written
186 + # (2) already read by this reader
187 + # try to read the next block
188 + self.current_idx = (self.current_idx +
189 + 1) % self.buffer.max_chunks
190 + if self.current_idx == start_index:
191 + # no block found
192 + if time.time(
193 + ) - start_time > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning: # noqa
194 + logger.warning(
195 + "No available block found in %s second. ",
196 + VLLM_RINGBUFFER_WARNING_INTERVAL)
197 + n_warning += 1
198 + # wait for a while (0.1 us)
199 + time.sleep(1e-7)
200 + continue
201 + # found a block that is not read by this reader
202 + # let caller read from the buffer
203 + with self.buffer.get_data(self.current_idx) as buf:
204 + yield buf
205 +
206 + # caller has read from the buffer
207 + # set the read flag
208 + metadata_buffer[self.reader_rank + 1] = 1
209 + break
210 +
211 + def enqueue(self, obj):
212 + assert self._is_writer, "Only writers can enqueue"
213 + serialized_obj = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
214 + if len(serialized_obj) > self.buffer.max_chunk_bytes:
215 + raise RuntimeError(
216 + f"{len(serialized_obj)=} larger than the allowed value "
217 + f"{self.buffer.max_chunk_bytes},"
218 + "Please increase the max_chunk_bytes parameter.")
219 + with self.acquire_write() as buf:
220 + buf[:len(serialized_obj)] = serialized_obj
221 +
222 + def dequeue(self):
223 + assert self._is_reader, "Only readers can dequeue"
224 + with self.acquire_read() as buf:
225 + # no need to know the size of serialized object
226 + # pickle format itself contains the size information internally
227 + # see https://docs.python.org/3/library/pickle.html
228 + obj = pickle.loads(buf)
229 + return obj
230 +
231 + def broadcast_object(self, obj=None):
232 + if self._is_writer:
233 + self.enqueue(obj)
234 + return obj
235 + else:
236 + return self.dequeue()
237 +
238 + def create_from_process_group(pg: ProcessGroup,
239 + max_chunk_bytes,
240 + max_chunks,
241 + writer_rank=0) -> "ShmRingBufferIO":
242 + group_rank = dist.get_rank(pg)
243 + group_world_size = dist.get_world_size(pg)
244 + ranks_inside_group = list(range(group_world_size))
245 + global_ranks = dist.get_process_group_ranks(pg)
246 + n_reader = group_world_size - 1
247 + buffer: ShmRingBuffer
248 + if group_rank == writer_rank:
249 + buffer = ShmRingBuffer(n_reader, max_chunk_bytes, max_chunks)
250 + dist.broadcast_object_list([buffer], src=global_ranks[writer_rank])
251 + dist.barrier(pg)
252 + return ShmRingBufferIO(buffer, -1)
253 + else:
254 + recv = [None]
255 + dist.broadcast_object_list(recv, src=global_ranks[writer_rank])
256 + dist.barrier(pg)
257 + buffer = recv[0] # type: ignore
258 + rest_ranks = [r for r in ranks_inside_group if r != writer_rank]
259 + return ShmRingBufferIO(buffer, rest_ranks.index(group_rank))
vllm/distributed/parallel_state.py
+35 -9
@@ -98,6 +98,7 @@ class GroupCoordinator:
98 # communicators are only created for world size > 1
99 pynccl_comm: Optional[Any] # PyNccl communicator
100 ca_comm: Optional[Any] # Custom allreduce communicator
101 + shm_broadcaster: Optional[Any] # shared memory broadcaster
102
103 def __init__(
104 self,
@@ -162,6 +163,13 @@ class GroupCoordinator:
163 else:
164 self.ca_comm = None
165
166 + from vllm.distributed.device_communicators.shm_broadcast import (
167 + ShmRingBufferIO)
168 + self.shm_broadcaster: Optional[ShmRingBufferIO] = None
169 + if self.world_size > 1 and is_in_the_same_node(self.cpu_group):
170 + self.shm_broadcaster = ShmRingBufferIO.create_from_process_group(
171 + self.cpu_group, 1 << 20, 6)
172 +
173 @property
174 def first_rank(self):
175 """Return the global rank of the first process in the group"""
@@ -324,6 +332,30 @@ class GroupCoordinator:
332 group=self.device_group)
333 return input_
334
335 + def broadcast_object(self, obj: Optional[Any] = None, src: int = 0):
336 + """Broadcast the input object.
337 + NOTE: `src` is the local rank of the source rank.
338 + """
339 + assert src < self.world_size, f"Invalid src rank ({src})"
340 +
341 + # Bypass the function if we are using only 1 GPU.
342 + if self.world_size == 1:
343 + return obj
344 + if self.shm_broadcaster is not None:
345 + assert src == 0, "Shared memory broadcaster only supports src=0"
346 + return self.shm_broadcaster.broadcast_object(obj)
347 + if self.rank_in_group == src:
348 + torch.distributed.broadcast_object_list([obj],
349 + src=self.ranks[src],
350 + group=self.cpu_group)
351 + return obj
352 + else:
353 + recv = [None]
354 + torch.distributed.broadcast_object_list(recv,
355 + src=self.ranks[src],
356 + group=self.cpu_group)
357 + return recv[0]
358 +
359 def broadcast_object_list(self,
360 obj_list: List[Any],
361 src: int = 0,
@@ -371,9 +403,7 @@ class GroupCoordinator:
403 # `metadata_list` lives in CPU memory.
404 # `broadcast_object_list` has serialization & deserialization,
405 # all happening on CPU. Therefore, we can use the CPU group.
374 - torch.distributed.broadcast_object_list([metadata_list],
375 - src=src,
376 - group=metadata_group)
406 + self.broadcast_object(metadata_list, src=src)
407 async_handles = []
408 for tensor in tensor_list:
409 if tensor.numel() == 0:
@@ -396,14 +426,10 @@ class GroupCoordinator:
426 async_handle.wait()
427
428 else:
399 - recv_metadata_list = [None]
400 - torch.distributed.broadcast_object_list(recv_metadata_list,
401 - src=src,
402 - group=metadata_group)
403 - assert recv_metadata_list[0] is not None
429 + metadata_list = self.broadcast_object(None, src=src)
430 tensor_dict = {}
431 async_handles = []
406 - for key, value in recv_metadata_list[0]:
432 + for key, value in metadata_list:
433 if isinstance(value, TensorMetadata):
434 tensor = torch.empty(value.size,
435 dtype=value.dtype,
vllm/envs.py
+5
@@ -5,6 +5,7 @@ if TYPE_CHECKING:
5 VLLM_HOST_IP: str = ""
6 VLLM_PORT: Optional[int] = None
7 VLLM_USE_MODELSCOPE: bool = False
8 + VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60
9 VLLM_INSTANCE_ID: Optional[str] = None
10 VLLM_NCCL_SO_PATH: Optional[str] = None
11 LD_LIBRARY_PATH: Optional[str] = None
@@ -114,6 +115,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
115 "VLLM_INSTANCE_ID":
116 lambda: os.environ.get("VLLM_INSTANCE_ID", None),
117
118 + # Interval in seconds to log a warning message when the ring buffer is full
119 + "VLLM_RINGBUFFER_WARNING_INTERVAL":
120 + lambda: int(os.environ.get("VLLM_RINGBUFFER_WARNING_INTERVAL", "60")),
121 +
122 # path to cudatoolkit home directory, under which should be bin, include,
123 # and lib directories.
124 "CUDA_HOME":