A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[core][distributed] fix custom allreduce in pytorch 2.5 (#9815)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Oct 29, 2024 at 17:06 UTC
1ab6f6b4ad5c4aac6ee72e51b7f6712098f9ccff
1 file changed
+13
-1
vllm/distributed/device_communicators/custom_all_reduce.py
+13
-1
@@ -191,8 +191,20 @@ class CustomAllreduce:
191
192
def _get_ipc_meta(self, inp: torch.Tensor):
193
data = inp.untyped_storage()._share_cuda_()
194
+ handle = data[1]
195
+ # https://github.com/pytorch/pytorch/pull/130890 changes
196
+ # the binary format of the ipc handle
197
+ # it starts from pytorch 2.5
198
+ if len(handle) > 64:
199
+ assert len(handle) == 66
200
+ # only support SHAREABLE_HANDLE_VERSION = 1
201
+ assert int(handle[0]) == 1
202
+ # only support SHAREABLE_CUDA_MALLOC = 'c'
203
+ assert handle[1] == ord("c")
204
+ handle = handle[2:]
205
+ # TODO: support expandable segment
206
shard_data = (
195
- data[1], # ipc handle to base ptr
207
+ handle, # ipc handle to base ptr
208
data[3], # offset of base ptr
209
)
210
return self._gather_ipc_meta(shard_data)