@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ab6f6b4ad
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[core][distributed] fix custom allreduce in pytorch 2.5 (#9815)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Oct 29, 2024 at 17:06 UTC 1ab6f6b4ad5c4aac6ee72e51b7f6712098f9ccff
1 file changed +13 -1
vllm/distributed/device_communicators/custom_all_reduce.py
+13 -1
@@ -191,8 +191,20 @@ class CustomAllreduce:
191
192 def _get_ipc_meta(self, inp: torch.Tensor):
193 data = inp.untyped_storage()._share_cuda_()
194 + handle = data[1]
195 + # https://github.com/pytorch/pytorch/pull/130890 changes
196 + # the binary format of the ipc handle
197 + # it starts from pytorch 2.5
198 + if len(handle) > 64:
199 + assert len(handle) == 66
200 + # only support SHAREABLE_HANDLE_VERSION = 1
201 + assert int(handle[0]) == 1
202 + # only support SHAREABLE_CUDA_MALLOC = 'c'
203 + assert handle[1] == ord("c")
204 + handle = handle[2:]
205 + # TODO: support expandable segment
206 shard_data = (
195 - data[1], # ipc handle to base ptr
207 + handle, # ipc handle to base ptr
208 data[3], # offset of base ptr
209 )
210 return self._gather_ipc_meta(shard_data)