@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f10797c0ce
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix][XPU] Fix xpu tp by introducing XpuCommunicator (#10144)

Signed-off-by: yan ma <yan.ma@intel.com>

Yan Ma committed Nov 8, 2024 at 17:41 UTC f10797c0ce4533412d41842180ca792ad07df11c
2 files changed +65 -22
vllm/distributed/device_communicators/xpu_communicator.py new
+47
@@ -0,0 +1,47 @@
1 +import torch
2 +import torch.distributed as dist
3 +from torch.distributed import ProcessGroup
4 +
5 +from vllm.platforms import current_platform
6 +
7 +
8 +class XpuCommunicator:
9 +
10 + def __init__(self, group: ProcessGroup):
11 + if not current_platform.is_xpu():
12 + self.disabled = True
13 + return
14 + self.disabled = False
15 + self.group = group
16 + self.world_size = dist.get_world_size(self.group)
17 +
18 + def all_reduce(self, x: torch.Tensor) -> torch.Tensor:
19 + dist.all_reduce(x, group=self.group)
20 + return x
21 +
22 + def gather(self,
23 + input_: torch.Tensor,
24 + rank_in_group: int,
25 + dst: int = 0,
26 + dim: int = -1):
27 + # For xpu path, gather doesn't work properly together with ray
28 + # cluster so we use all_gather instead for now.
29 + input_size = input_.size()
30 + # Allocate output tensor.
31 + output_tensor = torch.empty((self.world_size, ) + input_size,
32 + dtype=input_.dtype,
33 + device=input_.device)
34 + # All-gather.
35 + torch.distributed.all_gather_into_tensor(output_tensor,
36 + input_,
37 + group=self.group)
38 + if rank_in_group == dst:
39 + # Reshape
40 + output_tensor = output_tensor.movedim(0, dim)
41 + output_tensor = output_tensor.reshape(input_size[:dim] +
42 + (self.world_size *
43 + input_size[dim], ) +
44 + input_size[dim + 1:])
45 + else:
46 + output_tensor = None
47 + return output_tensor
vllm/distributed/parallel_state.py
+18 -22
@@ -177,6 +177,7 @@ class GroupCoordinator:
177 use_custom_allreduce: bool,
178 use_tpu_communicator: bool,
179 use_hpu_communicator: bool,
180 + use_xpu_communicator: bool,
181 use_message_queue_broadcaster: bool = False,
182 group_name: Optional[str] = None,
183 ):
@@ -214,6 +215,7 @@ class GroupCoordinator:
215 self.use_custom_allreduce = use_custom_allreduce
216 self.use_tpu_communicator = use_tpu_communicator
217 self.use_hpu_communicator = use_hpu_communicator
218 + self.use_xpu_communicator = use_xpu_communicator
219
220 # lazy import to avoid documentation build error
221 from vllm.distributed.device_communicators.custom_all_reduce import (
@@ -248,6 +250,12 @@ class GroupCoordinator:
250 if use_hpu_communicator and self.world_size > 1:
251 self.hpu_communicator = HpuCommunicator(group=self.device_group)
252
253 + from vllm.distributed.device_communicators.xpu_communicator import (
254 + XpuCommunicator)
255 + self.xpu_communicator: Optional[XpuCommunicator]
256 + if use_xpu_communicator and self.world_size > 1:
257 + self.xpu_communicator = XpuCommunicator(group=self.device_group)
258 +
259 from vllm.distributed.device_communicators.shm_broadcast import (
260 MessageQueue)
261 self.mq_broadcaster: Optional[MessageQueue] = None
@@ -373,6 +381,10 @@ class GroupCoordinator:
381 not self.hpu_communicator.disabled:
382 return self.hpu_communicator.all_reduce(input_)
383
384 + if self.xpu_communicator is not None and \
385 + not self.xpu_communicator.disabled:
386 + return self.xpu_communicator.all_reduce(input_)
387 +
388 if self.ca_comm is not None and \
389 not self.ca_comm.disabled and \
390 self.ca_comm.should_custom_ar(input_):
@@ -459,28 +471,10 @@ class GroupCoordinator:
471 if dim < 0:
472 # Convert negative dim to positive.
473 dim += input_.dim()
462 - # For xpu path, gather doesn't work properly together with ray
463 - # cluster so we use all_gather instead for now.
464 - if current_platform.is_xpu():
465 - input_size = input_.size()
466 - # Allocate output tensor.
467 - output_tensor = torch.empty((world_size, ) + input_size,
468 - dtype=input_.dtype,
469 - device=input_.device)
470 - # All-gather.
471 - torch.distributed.all_gather_into_tensor(output_tensor,
472 - input_,
473 - group=self.device_group)
474 - if self.rank_in_group == dst:
475 - # Reshape
476 - output_tensor = output_tensor.movedim(0, dim)
477 - output_tensor = output_tensor.reshape(input_size[:dim] +
478 - (world_size *
479 - input_size[dim], ) +
480 - input_size[dim + 1:])
481 - else:
482 - output_tensor = None
483 - return output_tensor
474 + if self.xpu_communicator is not None and \
475 + not self.xpu_communicator.disabled:
476 + return self.xpu_communicator.gather(input_, self.rank_in_group,
477 + dst, dim)
478 # Allocate output tensor.
479 if self.rank_in_group == dst:
480 gather_list = [torch.empty_like(input_) for _ in range(world_size)]
@@ -896,6 +890,7 @@ def init_world_group(ranks: List[int], local_rank: int,
890 use_custom_allreduce=False,
891 use_tpu_communicator=False,
892 use_hpu_communicator=False,
893 + use_xpu_communicator=False,
894 group_name="world",
895 )
896
@@ -918,6 +913,7 @@ def init_model_parallel_group(
913 use_custom_allreduce=use_custom_allreduce,
914 use_tpu_communicator=True,
915 use_hpu_communicator=True,
916 + use_xpu_communicator=True,
917 use_message_queue_broadcaster=use_message_queue_broadcaster,
918 group_name=group_name,
919 )