@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 515080ad2f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[bugfix][distributed] fix shm broadcast when the queue size is full (#5801)

youkaichao committed Jun 25, 2024 at 21:56 UTC 515080ad2fd93cc8e363ff43b90a9df18cfd71ff
2 files changed +76 -46
tests/distributed/test_shm_broadcast.py
+33 -16
@@ -1,7 +1,9 @@
1 import multiprocessing
2 import random
3 import time
4 +from typing import List
5
6 +import numpy as np
7 import torch.distributed as dist
8
9 from vllm.distributed.device_communicators.shm_broadcast import (
@@ -9,6 +11,14 @@ from vllm.distributed.device_communicators.shm_broadcast import (
11 from vllm.utils import update_environment_variables
12
13
14 +def get_arrays(n: int, seed: int = 0) -> List[np.ndarray]:
15 + np.random.seed(seed)
16 + sizes = np.random.randint(1, 10_000, n)
17 + # on average, each array will have 5k elements
18 + # with int64, each array will have 40kb
19 + return [np.random.randint(1, 100, i) for i in sizes]
20 +
21 +
22 def distributed_run(fn, world_size):
23 number_of_processes = world_size
24 processes = []
@@ -47,24 +57,31 @@ def worker_fn_wrapper(fn):
57 def worker_fn():
58 writer_rank = 2
59 broadcaster = ShmRingBufferIO.create_from_process_group(
50 - dist.group.WORLD, 1024, 2, writer_rank)
60 + dist.group.WORLD, 1024 * 1024, 2, writer_rank)
61 + if dist.get_rank() == writer_rank:
62 + seed = random.randint(0, 1000)
63 + dist.broadcast_object_list([seed], writer_rank)
64 + else:
65 + recv = [None]
66 + dist.broadcast_object_list(recv, writer_rank)
67 + seed = recv[0] # type: ignore
68 + dist.barrier()
69 + # in case we find a race condition
70 + # print the seed so that we can reproduce the error
71 + print(f"Rank {dist.get_rank()} got seed {seed}")
72 + # test broadcasting with about 400MB of data
73 + N = 10_000
74 if dist.get_rank() == writer_rank:
52 - time.sleep(random.random())
53 - broadcaster.broadcast_object(0)
54 - time.sleep(random.random())
55 - broadcaster.broadcast_object({})
56 - time.sleep(random.random())
57 - broadcaster.broadcast_object([])
75 + arrs = get_arrays(N, seed)
76 + for x in arrs:
77 + broadcaster.broadcast_object(x)
78 + time.sleep(random.random() / 1000)
79 else:
59 - time.sleep(random.random())
60 - a = broadcaster.broadcast_object(None)
61 - time.sleep(random.random())
62 - b = broadcaster.broadcast_object(None)
63 - time.sleep(random.random())
64 - c = broadcaster.broadcast_object(None)
65 - assert a == 0
66 - assert b == {}
67 - assert c == []
80 + arrs = get_arrays(N, seed)
81 + for x in arrs:
82 + y = broadcaster.broadcast_object(None)
83 + assert np.array_equal(x, y)
84 + time.sleep(random.random() / 1000)
85 dist.barrier()
86
87
vllm/distributed/device_communicators/shm_broadcast.py
+43 -30
@@ -14,6 +14,12 @@ from vllm.logger import init_logger
14
15 VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
16
17 +# time to wait if the queue is full or empty
18 +# if we sleep for too short, it will consume too much CPU
19 +# if we sleep for too long, it will slow down the writer/reader
20 +# 0.1 us is a good balance
21 +RINGBUFFER_SLEEP_INTERVAL = 1e-7
22 +
23 logger = init_logger(__name__)
24
25
@@ -145,8 +151,7 @@ class ShmRingBufferIO:
151 @contextmanager
152 def acquire_write(self):
153 assert self._is_writer, "Only writers can acquire write"
148 - start_index = self.current_idx
149 - start_time = time.time()
154 + start_time = time.monotonic()
155 n_warning = 1
156 while True:
157 with self.buffer.get_metadata(self.current_idx) as metadata_buffer:
@@ -154,19 +159,21 @@ class ShmRingBufferIO:
159 written_flag = metadata_buffer[0]
160 if written_flag and read_count != self.buffer.n_reader:
161 # this block is written and not read by all readers
157 - # try to write to the next block
158 - self.current_idx = (self.current_idx +
159 - 1) % self.buffer.max_chunks
160 - if self.current_idx == start_index:
161 - # no empty block found
162 - if time.time(
163 - ) - start_time > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning: # noqa
164 - logger.warning(
165 - "No available block found in %s second. ",
166 - VLLM_RINGBUFFER_WARNING_INTERVAL)
167 - n_warning += 1
168 - # wait for a while (0.1 us)
169 - time.sleep(1e-7)
162 + # for writers, `self.current_idx` is the next block to write
163 + # if this block is not ready to write,
164 + # we need to wait until it is read by all readers
165 +
166 + # wait for a while
167 + time.sleep(RINGBUFFER_SLEEP_INTERVAL)
168 +
169 + # if we wait for a long time, we should warn the user
170 + if time.monotonic(
171 + ) - start_time > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning: # noqa
172 + logger.warning(
173 + "No available block found in %s second. ",
174 + VLLM_RINGBUFFER_WARNING_INTERVAL)
175 + n_warning += 1
176 +
177 continue
178 # found a block that is either
179 # (1) not written
@@ -188,13 +195,14 @@ class ShmRingBufferIO:
195 metadata_buffer[i] = 0
196 # mark the block as written
197 metadata_buffer[0] = 1
198 + self.current_idx = (self.current_idx +
199 + 1) % self.buffer.max_chunks
200 break
201
202 @contextmanager
203 def acquire_read(self):
204 assert self._is_reader, "Only readers can acquire read"
196 - start_index = self.current_idx
197 - start_time = time.time()
205 + start_time = time.monotonic()
206 n_warning = 1
207 while True:
208 with self.buffer.get_metadata(self.current_idx) as metadata_buffer:
@@ -204,19 +212,22 @@ class ShmRingBufferIO:
212 # this block is either
213 # (1) not written
214 # (2) already read by this reader
207 - # try to read the next block
208 - self.current_idx = (self.current_idx +
209 - 1) % self.buffer.max_chunks
210 - if self.current_idx == start_index:
211 - # no block found
212 - if time.time(
213 - ) - start_time > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning: # noqa
214 - logger.warning(
215 - "No available block found in %s second. ",
216 - VLLM_RINGBUFFER_WARNING_INTERVAL)
217 - n_warning += 1
218 - # wait for a while (0.1 us)
219 - time.sleep(1e-7)
215 +
216 + # for readers, `self.current_idx` is the next block to read
217 + # if this block is not ready,
218 + # we need to wait until it is written
219 +
220 + # wait for a while
221 + time.sleep(RINGBUFFER_SLEEP_INTERVAL)
222 +
223 + # if we wait for a long time, we should warn the user
224 + if time.monotonic(
225 + ) - start_time > VLLM_RINGBUFFER_WARNING_INTERVAL * n_warning: # noqa
226 + logger.warning(
227 + "No available block found in %s second. ",
228 + VLLM_RINGBUFFER_WARNING_INTERVAL)
229 + n_warning += 1
230 +
231 continue
232 # found a block that is not read by this reader
233 # let caller read from the buffer
@@ -226,6 +237,8 @@ class ShmRingBufferIO:
237 # caller has read from the buffer
238 # set the read flag
239 metadata_buffer[self.reader_rank + 1] = 1
240 + self.current_idx = (self.current_idx +
241 + 1) % self.buffer.max_chunks
242 break
243
244 def enqueue(self, obj):