A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix][XPU] Fix xpu tp by introducing XpuCommunicator (#10144)
Signed-off-by: yan ma <yan.ma@intel.com>
Yan Ma committed
Nov 8, 2024 at 17:41 UTC
f10797c0ce4533412d41842180ca792ad07df11c
2 files changed
+65
-22
vllm/distributed/device_communicators/xpu_communicator.py
new
+47
@@ -0,0 +1,47 @@
1
+import torch
2
+import torch.distributed as dist
3
+from torch.distributed import ProcessGroup
4
+
5
+from vllm.platforms import current_platform
6
+
7
+
8
+class XpuCommunicator:
9
+
10
+ def __init__(self, group: ProcessGroup):
11
+ if not current_platform.is_xpu():
12
+ self.disabled = True
13
+ return
14
+ self.disabled = False
15
+ self.group = group
16
+ self.world_size = dist.get_world_size(self.group)
17
+
18
+ def all_reduce(self, x: torch.Tensor) -> torch.Tensor:
19
+ dist.all_reduce(x, group=self.group)
20
+ return x
21
+
22
+ def gather(self,
23
+ input_: torch.Tensor,
24
+ rank_in_group: int,
25
+ dst: int = 0,
26
+ dim: int = -1):
27
+ # For xpu path, gather doesn't work properly together with ray
28
+ # cluster so we use all_gather instead for now.
29
+ input_size = input_.size()
30
+ # Allocate output tensor.
31
+ output_tensor = torch.empty((self.world_size, ) + input_size,
32
+ dtype=input_.dtype,
33
+ device=input_.device)
34
+ # All-gather.
35
+ torch.distributed.all_gather_into_tensor(output_tensor,
36
+ input_,
37
+ group=self.group)
38
+ if rank_in_group == dst:
39
+ # Reshape
40
+ output_tensor = output_tensor.movedim(0, dim)
41
+ output_tensor = output_tensor.reshape(input_size[:dim] +
42
+ (self.world_size *
43
+ input_size[dim], ) +
44
+ input_size[dim + 1:])
45
+ else:
46
+ output_tensor = None
47
+ return output_tensor
vllm/distributed/parallel_state.py
+18
-22
@@ -177,6 +177,7 @@ class GroupCoordinator:
177
use_custom_allreduce: bool,
178
use_tpu_communicator: bool,
179
use_hpu_communicator: bool,
180
+ use_xpu_communicator: bool,
181
use_message_queue_broadcaster: bool = False,
182
group_name: Optional[str] = None,
183
):
@@ -214,6 +215,7 @@ class GroupCoordinator:
215
self.use_custom_allreduce = use_custom_allreduce
216
self.use_tpu_communicator = use_tpu_communicator
217
self.use_hpu_communicator = use_hpu_communicator
218
+ self.use_xpu_communicator = use_xpu_communicator
219
220
# lazy import to avoid documentation build error
221
from vllm.distributed.device_communicators.custom_all_reduce import (
@@ -248,6 +250,12 @@ class GroupCoordinator:
250
if use_hpu_communicator and self.world_size > 1:
251
self.hpu_communicator = HpuCommunicator(group=self.device_group)
252
253
+ from vllm.distributed.device_communicators.xpu_communicator import (
254
+ XpuCommunicator)
255
+ self.xpu_communicator: Optional[XpuCommunicator]
256
+ if use_xpu_communicator and self.world_size > 1:
257
+ self.xpu_communicator = XpuCommunicator(group=self.device_group)
258
+
259
from vllm.distributed.device_communicators.shm_broadcast import (
260
MessageQueue)
261
self.mq_broadcaster: Optional[MessageQueue] = None
@@ -373,6 +381,10 @@ class GroupCoordinator:
381
not self.hpu_communicator.disabled:
382
return self.hpu_communicator.all_reduce(input_)
383
384
+ if self.xpu_communicator is not None and \
385
+ not self.xpu_communicator.disabled:
386
+ return self.xpu_communicator.all_reduce(input_)
387
+
388
if self.ca_comm is not None and \
389
not self.ca_comm.disabled and \
390
self.ca_comm.should_custom_ar(input_):
@@ -459,28 +471,10 @@ class GroupCoordinator:
471
if dim < 0:
472
# Convert negative dim to positive.
473
dim += input_.dim()
462
- # For xpu path, gather doesn't work properly together with ray
463
- # cluster so we use all_gather instead for now.
464
- if current_platform.is_xpu():
465
- input_size = input_.size()
466
- # Allocate output tensor.
467
- output_tensor = torch.empty((world_size, ) + input_size,
468
- dtype=input_.dtype,
469
- device=input_.device)
470
- # All-gather.
471
- torch.distributed.all_gather_into_tensor(output_tensor,
472
- input_,
473
- group=self.device_group)
474
- if self.rank_in_group == dst:
475
- # Reshape
476
- output_tensor = output_tensor.movedim(0, dim)
477
- output_tensor = output_tensor.reshape(input_size[:dim] +
478
- (world_size *
479
- input_size[dim], ) +
480
- input_size[dim + 1:])
481
- else:
482
- output_tensor = None
483
- return output_tensor
474
+ if self.xpu_communicator is not None and \
475
+ not self.xpu_communicator.disabled:
476
+ return self.xpu_communicator.gather(input_, self.rank_in_group,
477
+ dst, dim)
478
# Allocate output tensor.
479
if self.rank_in_group == dst:
480
gather_list = [torch.empty_like(input_) for _ in range(world_size)]
@@ -896,6 +890,7 @@ def init_world_group(ranks: List[int], local_rank: int,
890
use_custom_allreduce=False,
891
use_tpu_communicator=False,
892
use_hpu_communicator=False,
893
+ use_xpu_communicator=False,
894
group_name="world",
895
)
896
@@ -918,6 +913,7 @@ def init_model_parallel_group(
913
use_custom_allreduce=use_custom_allreduce,
914
use_tpu_communicator=True,
915
use_hpu_communicator=True,
916
+ use_xpu_communicator=True,
917
use_message_queue_broadcaster=use_message_queue_broadcaster,
918
group_name=group_name,
919
)