@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a08df8322e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[TPU] Support multi-host inference (#7457)

Woosuk Kwon committed Aug 13, 2024 at 16:31 UTC a08df8322e2e5f27cb696bc2c0d8cdabe185878b
2 files changed +11 -4
docs/source/getting_started/tpu-installation.rst
+1 -1
@@ -8,7 +8,7 @@ vLLM supports Google Cloud TPUs using PyTorch XLA.
8 Requirements
9 ------------
10
11 -* Google Cloud TPU VM (single host)
11 +* Google Cloud TPU VM (single & multi host)
12 * TPU versions: v5e, v5p, v4
13 * Python: 3.10
14
vllm/distributed/device_communicators/tpu_communicator.py
+10 -3
@@ -1,3 +1,4 @@
1 +import ray
2 import torch
3 import torch.distributed as dist
4 from torch.distributed import ProcessGroup
@@ -18,9 +19,15 @@ class TpuCommunicator:
19 return
20 self.disabled = False
21
21 - local_rank = dist.get_rank(group)
22 - world_size = dist.get_world_size(group)
23 - pjrt.initialize_multiprocess(local_rank, world_size)
22 + # NOTE(woosuk): When using TP > 1 on TPUs, every TPU on the same node
23 + # must be used together. Therefore, the local rank and world size can
24 + # be simply calculated as follows.
25 + global_rank = dist.get_rank(group)
26 + global_world_size = dist.get_world_size(group)
27 + num_nodes = len(ray.nodes())
28 + local_world_size = global_world_size // num_nodes
29 + local_rank = global_rank % local_world_size
30 + pjrt.initialize_multiprocess(local_rank, local_world_size)
31 xr._init_world_size_ordinal()
32
33 def all_reduce(self, x: torch.Tensor) -> torch.Tensor: