@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 832ea88fcb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[core][distributed] improve shared memory broadcast (#5754)

youkaichao committed Jun 22, 2024 at 10:00 UTC 832ea88fcb4819037b685fb47b3a0de37f2804d3
1 file changed +32 -10
vllm/distributed/device_communicators/shm_broadcast.py
+32 -10
@@ -48,6 +48,26 @@ class ShmRingBuffer:
48 | written_flag | reader0_flag | reader1_flag | ... | readerN_flag |
49 +--------------+--------------+--------------+-----+--------------+
50
51 + The state of metadata is as follows:
52 +
53 + (case 1) 0???...???: the block is not written yet, cannot read, can write
54 + (case 2) 1000...000: the block is just written, can read, cannot write
55 + (case 3) 1???...???: the block is written and read by some readers, can read if not read, cannot write
56 + (case 4) 1111...111: the block is written and read by all readers, cannot read, can write
57 +
58 + State transition for readers:
59 +
60 + When a reader finds a block that it can read (case 2 or 3), it can yield the block for caller to read.
61 + Only after the caller finishes reading the block, the reader can mark the block as read.
62 + Readers only mark the block as read (from 0 to 1), the writer marks the block as ready to read (from 1 to 0).
63 +
64 + State transition for writer:
65 +
66 + When the writer writes to a block (case 1 or 4), it first resets the written flag to 0, converting either case
67 + to case 1. Then it can yield the block for caller to write. After the caller finishes writing the block, the writer
68 + can reset the reader flags to 0, and mark the block as written (from 0 to 1).
69 + NOTE: the order is important here, first reset the reader flags (so that we are still in case 1), then mark the block as written. The state transition is atomic. If we do it in the reverse order, it will go through case 3 and then back to case 2, and readers might read the intermediate case 3, which is not correct.
70 +
71 During creation, `name` is None and the buffer is created. We can pass the
72 created object to other processes by pickling it. The other processes will
73 get the name of the shared memory and open it, so that they can access the
@@ -81,10 +101,6 @@ class ShmRingBuffer:
101 lambda *args, **kwargs: None):
102 self.shared_memory = shared_memory.SharedMemory(name=name)
103 assert self.shared_memory.size == self.total_bytes_of_buffer
84 - with memoryview(self.shared_memory.buf[self.metadata_offset:]
85 - ) as metadata_buffer:
86 - tensor = torch.frombuffer(metadata_buffer, dtype=torch.uint8)
87 - assert torch.all(tensor == 0)
104
105 def __reduce__(self):
106 return (
@@ -163,11 +179,15 @@ class ShmRingBufferIO:
179 yield buf
180
181 # caller has written to the buffer
166 - # mark the block as written
167 - metadata_buffer[0] = 1
182 + # NOTE: order is important here
183 + # first set the read flags to 0
184 + # then set the written flag to 1
185 + # otherwise, the readers may think they already read the block
186 for i in range(1, self.buffer.n_reader + 1):
187 # set read flag to 0, meaning it is not read yet
188 metadata_buffer[i] = 0
189 + # mark the block as written
190 + metadata_buffer[0] = 1
191 break
192
193 @contextmanager
@@ -247,13 +267,15 @@ class ShmRingBufferIO:
267 buffer: ShmRingBuffer
268 if group_rank == writer_rank:
269 buffer = ShmRingBuffer(n_reader, max_chunk_bytes, max_chunks)
250 - dist.broadcast_object_list([buffer], src=global_ranks[writer_rank])
251 - dist.barrier(pg)
270 + dist.broadcast_object_list([buffer],
271 + src=global_ranks[writer_rank],
272 + group=pg)
273 return ShmRingBufferIO(buffer, -1)
274 else:
275 recv = [None]
255 - dist.broadcast_object_list(recv, src=global_ranks[writer_rank])
256 - dist.barrier(pg)
276 + dist.broadcast_object_list(recv,
277 + src=global_ranks[writer_rank],
278 + group=pg)
279 buffer = recv[0] # type: ignore
280 rest_ranks = [r for r in ranks_inside_group if r != writer_rank]
281 return ShmRingBufferIO(buffer, rest_ranks.index(group_rank))