A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[TPU] Support single and multi-host TPUs on GKE (#7613)
Richard Liu committed
Aug 30, 2024 at 00:27 UTC
2148441fd371faf3e90748b310fdb4500939e527
5 files changed
+74
-4
requirements-tpu.txt
+1
-1
@@ -4,4 +4,4 @@
4
# Dependencies for TPU
5
# Currently, the TPU backend uses a nightly version of PyTorch XLA.
6
# You can install the dependencies in Dockerfile.tpu.
7
-ray
7
+ray[default]
vllm/attention/backends/pallas.py
+4
-1
@@ -123,7 +123,10 @@ class PallasAttentionBackendImpl(AttentionImpl):
123
raise NotImplementedError("TPU version must be 4 or higher.")
124
125
self.megacore_mode = None
126
- tpu_type = torch_xla.tpu.get_tpu_env()["TYPE"].lower()
126
+ tpu_env = torch_xla.tpu.get_tpu_env()
127
+ tpu_type = tpu_env.get("TYPE") or tpu_env.get("ACCELERATOR_TYPE")
128
+ tpu_type = tpu_type.lower()
129
+
130
if "lite" not in tpu_type:
131
if self.num_kv_heads % 2 == 0:
132
self.megacore_mode = "kv_head"
vllm/distributed/device_communicators/tpu_communicator.py
+25
-2
@@ -1,3 +1,5 @@
1
+import os
2
+
3
import torch
4
import torch.distributed as dist
5
from torch.distributed import ProcessGroup
@@ -5,11 +7,12 @@ from torch.distributed import ProcessGroup
7
from vllm.platforms import current_platform
8
9
if current_platform.is_tpu():
8
- import ray
10
import torch_xla.core.xla_model as xm
11
import torch_xla.runtime as xr
12
from torch_xla._internal import pjrt
13
14
+ from vllm.executor import ray_utils
15
+
16
17
class TpuCommunicator:
18
@@ -24,9 +27,29 @@ class TpuCommunicator:
27
# be simply calculated as follows.
28
global_rank = dist.get_rank(group)
29
global_world_size = dist.get_world_size(group)
27
- num_nodes = len(ray.nodes())
30
+
31
+ # Calculate how many TPU nodes are in the current deployment. This
32
+ # is the Ray placement group if it is deployed with Ray. Default
33
+ # to the number of TPU nodes in the Ray cluster. The number of TPU
34
+ # nodes is computed by the total number of TPUs divided by the
35
+ # number of TPU accelerators per node, to account for clusters
36
+ # with both CPUs and TPUs.
37
+ num_nodes = ray_utils.get_num_tpu_nodes()
38
+ num_nodes_in_pg = ray_utils.get_num_nodes_in_placement_group()
39
+ if num_nodes_in_pg > 0:
40
+ num_nodes = num_nodes_in_pg
41
+
42
local_world_size = global_world_size // num_nodes
43
local_rank = global_rank % local_world_size
44
+
45
+ # Ensure environment variables are set for multihost deployments.
46
+ # On GKE, this is needed for libtpu and TPU driver to know which TPU
47
+ # chip is actually visible. Otherwise the TPU driver will fail to
48
+ # initialize because the number of devices would be different from
49
+ # the number of visible worker addresses.
50
+ os.environ["CLOUD_TPU_TASK_ID"] = str(global_rank)
51
+ os.environ["TPU_VISIBLE_CHIPS"] = str(local_rank)
52
+
53
pjrt.initialize_multiprocess(local_rank, local_world_size)
54
xr._init_world_size_ordinal()
55
vllm/executor/ray_tpu_executor.py
+15
@@ -71,6 +71,19 @@ class RayTPUExecutor(TPUExecutor):
71
worker_module_name = "vllm.worker.tpu_worker"
72
worker_class_name = "TPUWorker"
73
74
+ # GKE does not fetch environment information from metadata server
75
+ # and instead sets these from within the Ray process. Therefore we
76
+ # need to override the Ray environment variables manually.
77
+ override_env = {}
78
+ if "TPU_CHIPS_PER_HOST_BOUNDS" in os.environ:
79
+ override_env.update({
80
+ "TPU_CHIPS_PER_HOST_BOUNDS":
81
+ os.environ["TPU_CHIPS_PER_HOST_BOUNDS"]
82
+ })
83
+ if "TPU_HOST_BOUNDS" in os.environ:
84
+ override_env.update(
85
+ {"TPU_HOST_BOUNDS": os.environ["TPU_HOST_BOUNDS"]})
86
+
87
worker = ray.remote(
88
num_cpus=0,
89
resources={"TPU": 1},
@@ -81,6 +94,8 @@ class RayTPUExecutor(TPUExecutor):
94
worker_class_name=worker_class_name,
95
trust_remote_code=self.model_config.trust_remote_code,
96
)
97
+ if override_env:
98
+ worker.override_env_vars.remote(override_env)
99
100
worker_ip = ray.get(worker.get_node_ip.remote())
101
if worker_ip == driver_ip and self.driver_dummy_worker is None:
vllm/executor/ray_utils.py
+29
@@ -1,3 +1,4 @@
1
+import os
2
import time
3
from collections import defaultdict
4
from typing import Dict, List, Optional, Tuple, Union
@@ -84,6 +85,9 @@ try:
85
86
return output
87
88
+ def override_env_vars(self, vars: Dict[str, str]):
89
+ os.environ.update(vars)
90
+
91
ray_import_err = None
92
93
except ImportError as e:
@@ -291,3 +295,28 @@ def initialize_ray_cluster(
295
_verify_bundles(current_placement_group, parallel_config, device_str)
296
# Set the placement group in the parallel config
297
parallel_config.placement_group = current_placement_group
298
+
299
+
300
+def get_num_tpu_nodes() -> int:
301
+ from ray._private.accelerators import TPUAcceleratorManager
302
+ cluster_resources = ray.cluster_resources()
303
+ total_tpus = int(cluster_resources["TPU"])
304
+ tpus_per_node = TPUAcceleratorManager.get_current_node_num_accelerators()
305
+ assert total_tpus % tpus_per_node == 0
306
+ return total_tpus // tpus_per_node
307
+
308
+
309
+def get_num_nodes_in_placement_group() -> int:
310
+ pg_table = ray.util.placement_group_table()
311
+ current_pg = ray.util.get_current_placement_group()
312
+ num_nodes = 0
313
+
314
+ if current_pg:
315
+ nodes_in_pg = set()
316
+ for pg_key, pg in pg_table.items():
317
+ if pg_key == current_pg.id.hex():
318
+ for _, node in pg["bundles_to_node_id"].items():
319
+ nodes_in_pg.add(node)
320
+ num_nodes = len(nodes_in_pg)
321
+
322
+ return num_nodes