@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aa4867791e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc][TPU] Support TPU in initialize_ray_cluster (#6812)

Woosuk Kwon committed Jul 26, 2024 at 12:39 UTC aa4867791ecd73a5f55b7bad4d9372954e661fe4
1 file changed +21 -15
vllm/executor/ray_utils.py
+21 -15
@@ -3,7 +3,7 @@ from typing import List, Optional, Tuple
3 from vllm.config import ParallelConfig
4 from vllm.logger import init_logger
5 from vllm.sequence import ExecuteModelRequest
6 -from vllm.utils import get_ip, is_hip, is_xpu
6 +from vllm.utils import get_ip, is_hip, is_tpu, is_xpu
7 from vllm.worker.worker_base import WorkerWrapperBase
8
9 logger = init_logger(__name__)
@@ -93,32 +93,38 @@ def initialize_ray_cluster(
93 # Placement group is already set.
94 return
95
96 + device_str = "GPU" if not is_tpu() else "TPU"
97 # Create placement group for worker processes
98 current_placement_group = ray.util.get_current_placement_group()
99 if current_placement_group:
100 # We are in a placement group
101 bundles = current_placement_group.bundle_specs
102 # Verify that we can use the placement group.
102 - gpu_bundles = 0
103 + device_bundles = 0
104 for bundle in bundles:
104 - bundle_gpus = bundle.get("GPU", 0)
105 - if bundle_gpus > 1:
105 + bundle_devices = bundle.get(device_str, 0)
106 + if bundle_devices > 1:
107 raise ValueError(
107 - "Placement group bundle cannot have more than 1 GPU.")
108 - if bundle_gpus:
109 - gpu_bundles += 1
110 - if parallel_config.world_size > gpu_bundles:
108 + "Placement group bundle cannot have more than 1 "
109 + f"{device_str}.")
110 + if bundle_devices:
111 + device_bundles += 1
112 + if parallel_config.world_size > device_bundles:
113 raise ValueError(
112 - "The number of required GPUs exceeds the total number of "
113 - "available GPUs in the placement group.")
114 + f"The number of required {device_str}s exceeds the total "
115 + f"number of available {device_str}s in the placement group."
116 + f"Required number of devices: {parallel_config.world_size}. "
117 + f"Total number of devices: {device_bundles}.")
118 else:
115 - num_gpus_in_cluster = ray.cluster_resources().get("GPU", 0)
116 - if parallel_config.world_size > num_gpus_in_cluster:
119 + num_devices_in_cluster = ray.cluster_resources().get(device_str, 0)
120 + if parallel_config.world_size > num_devices_in_cluster:
121 raise ValueError(
118 - "The number of required GPUs exceeds the total number of "
119 - "available GPUs in the cluster.")
122 + f"The number of required {device_str}s exceeds the total "
123 + f"number of available {device_str}s in the placement group.")
124 # Create a new placement group
121 - placement_group_specs = ([{"GPU": 1}] * parallel_config.world_size)
125 + placement_group_specs = ([{
126 + device_str: 1
127 + }] * parallel_config.world_size)
128 current_placement_group = ray.util.placement_group(
129 placement_group_specs)
130 # Wait until PG is ready - this will block until all