@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d09b94ca58
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[TPU] Support collective communications in XLA devices (#6813)

Woosuk Kwon committed Jul 26, 2024 at 18:45 UTC d09b94ca588c6de1e627194264357e14460ae2eb
4 files changed +70 -2
vllm/distributed/device_communicators/tpu_communicator.py new
+30
@@ -0,0 +1,30 @@
1 +import torch
2 +import torch.distributed as dist
3 +from torch.distributed import ProcessGroup
4 +
5 +from vllm.platforms import current_platform
6 +
7 +if current_platform.is_tpu():
8 + import torch_xla.core.xla_model as xm
9 + from torch_xla._internal import pjrt
10 +
11 +
12 +class TpuCommunicator:
13 +
14 + def __init__(self, group: ProcessGroup):
15 + if not current_platform.is_tpu():
16 + self.disabled = True
17 + return
18 + self.disabled = False
19 +
20 + local_rank = dist.get_rank(group)
21 + world_size = dist.get_world_size(group)
22 + pjrt.initialize_multiprocess(local_rank, world_size)
23 + xm._init_world_size_ordinal()
24 +
25 + def all_reduce(self, x: torch.Tensor) -> torch.Tensor:
26 + return xm.all_reduce(xm.REDUCE_SUM, x)
27 +
28 + def all_gather(self, x: torch.Tensor, dim: int = -1) -> torch.Tensor:
29 + assert dim == -1, "TPUs only support dim=-1 for all-gather."
30 + return xm.all_gather(x, dim=dim)
vllm/distributed/parallel_state.py
+22
@@ -133,6 +133,7 @@ class GroupCoordinator:
133 torch_distributed_backend: Union[str, Backend],
134 use_pynccl: bool,
135 use_custom_allreduce: bool,
136 + use_tpu_communicator: bool,
137 use_message_queue_broadcaster: bool = False,
138 ):
139
@@ -164,6 +165,7 @@ class GroupCoordinator:
165
166 self.use_pynccl = use_pynccl
167 self.use_custom_allreduce = use_custom_allreduce
168 + self.use_tpu_communicator = use_tpu_communicator
169
170 # lazy import to avoid documentation build error
171 from vllm.distributed.device_communicators.custom_all_reduce import (
@@ -190,6 +192,12 @@ class GroupCoordinator:
192 else:
193 self.ca_comm = None
194
195 + from vllm.distributed.device_communicators.tpu_communicator import (
196 + TpuCommunicator)
197 + self.tpu_communicator: Optional[TpuCommunicator]
198 + if use_tpu_communicator and self.world_size > 1:
199 + self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
200 +
201 from vllm.distributed.device_communicators.shm_broadcast import (
202 MessageQueue)
203 self.mq_broadcaster: Optional[MessageQueue] = None
@@ -289,6 +297,12 @@ class GroupCoordinator:
297 # Bypass the function if we are using only 1 GPU.
298 if self.world_size == 1:
299 return input_
300 +
301 + # For TPUs, use TPU communicator.
302 + tpu_comm = self.tpu_communicator
303 + if tpu_comm is not None and not tpu_comm.disabled:
304 + return tpu_comm.all_reduce(input_)
305 +
306 if ca_comm is not None:
307 out = ca_comm.custom_all_reduce(input_)
308 if out is not None:
@@ -310,6 +324,12 @@ class GroupCoordinator:
324 return input_
325 assert -input_.dim() <= dim < input_.dim(), (
326 f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
327 +
328 + # For TPUs, use TPU communicator.
329 + tpu_comm = self.tpu_communicator
330 + if tpu_comm is not None and not tpu_comm.disabled:
331 + return tpu_comm.all_gather(input_, dim)
332 +
333 if dim < 0:
334 # Convert negative dim to positive.
335 dim += input_.dim()
@@ -727,6 +747,7 @@ def init_world_group(ranks: List[int], local_rank: int,
747 torch_distributed_backend=backend,
748 use_pynccl=False,
749 use_custom_allreduce=False,
750 + use_tpu_communicator=False,
751 )
752
753
@@ -745,6 +766,7 @@ def init_model_parallel_group(
766 torch_distributed_backend=backend,
767 use_pynccl=True,
768 use_custom_allreduce=use_custom_allreduce,
769 + use_tpu_communicator=True,
770 use_message_queue_broadcaster=use_message_queue_broadcaster,
771 )
772
vllm/lora/layers.py
+4
@@ -1067,6 +1067,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1067 def soft_cap(self):
1068 return self.base_layer.soft_cap
1069
1070 + @property
1071 + def use_gather(self):
1072 + return self.base_layer.use_gather
1073 +
1074 @property
1075 def org_vocab_size(self):
1076 return self.base_layer.org_vocab_size
vllm/model_executor/layers/logits_processor.py
+14 -2
@@ -5,10 +5,12 @@ from typing import Optional
5 import torch
6 import torch.nn as nn
7
8 -from vllm.distributed import tensor_model_parallel_gather
8 +from vllm.distributed import (tensor_model_parallel_all_gather,
9 + tensor_model_parallel_gather)
10 from vllm.model_executor.layers.vocab_parallel_embedding import (
11 VocabParallelEmbedding)
12 from vllm.model_executor.sampling_metadata import SamplingMetadata
13 +from vllm.platforms import current_platform
14
15
16 class LogitsProcessor(nn.Module):
@@ -39,6 +41,8 @@ class LogitsProcessor(nn.Module):
41 self.org_vocab_size = org_vocab_size or vocab_size
42 # Soft cap the logits. Used in Gemma 2.
43 self.soft_cap = soft_cap
44 + # Whether to use gather or all-gather to gather the logits.
45 + self.use_gather = not current_platform.is_tpu()
46
47 def forward(
48 self,
@@ -76,7 +80,15 @@ class LogitsProcessor(nn.Module):
80 logits = lm_head.linear_method.apply(lm_head,
81 hidden_states,
82 bias=embedding_bias)
79 - logits = tensor_model_parallel_gather(logits)
83 + if self.use_gather:
84 + logits = tensor_model_parallel_gather(logits)
85 + else:
86 + # Gather is not supported for some devices such as TPUs.
87 + # Use all-gather instead.
88 + # NOTE(woosuk): Here, the outputs of every device should not be None
89 + # because XLA requires strict SPMD among all devices. Every device
90 + # should execute the same operations after gathering the logits.
91 + logits = tensor_model_parallel_all_gather(logits)
92 # Remove paddings in vocab (if any).
93 if logits is not None:
94 logits = logits[:, :self.org_vocab_size]