A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[platform] add ray_device_key (#11948)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 13, 2025 at 16:20 UTC
89ce62a316e68c50121e74d5a832e0cb9a5101d1
9 files changed
+38
-8
vllm/executor/ray_utils.py
+13
-6
@@ -8,6 +8,7 @@ import msgspec
8
from vllm.config import ParallelConfig
9
from vllm.executor.msgspec_utils import decode_hook, encode_hook
10
from vllm.logger import init_logger
11
+from vllm.platforms import current_platform
12
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
13
from vllm.utils import get_ip
14
from vllm.worker.worker_base import WorkerWrapperBase
@@ -47,7 +48,12 @@ try:
48
49
def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
50
node_id = ray.get_runtime_context().get_node_id()
50
- gpu_ids = ray.get_gpu_ids()
51
+ device_key = current_platform.ray_device_key
52
+ if not device_key:
53
+ raise RuntimeError("current platform %s does not support ray.",
54
+ current_platform.device_name)
55
+ gpu_ids = ray.get_runtime_context().get_accelerator_ids(
56
+ )[device_key]
57
return node_id, gpu_ids
58
59
def execute_model_spmd(
@@ -249,11 +255,12 @@ def initialize_ray_cluster(
255
# Placement group is already set.
256
return
257
252
- device_str = "GPU"
253
- if current_platform.is_tpu():
254
- device_str = "TPU"
255
- elif current_platform.is_hpu():
256
- device_str = 'HPU'
258
+ device_str = current_platform.ray_device_key
259
+ if not device_str:
260
+ raise ValueError(
261
+ f"current platform {current_platform.device_name} does not "
262
+ "support ray.")
263
+
264
# Create placement group for worker processes
265
current_placement_group = ray.util.get_current_placement_group()
266
if current_placement_group:
vllm/platforms/cuda.py
+1
@@ -77,6 +77,7 @@ class CudaPlatformBase(Platform):
77
device_name: str = "cuda"
78
device_type: str = "cuda"
79
dispatch_key: str = "CUDA"
80
+ ray_device_key: str = "GPU"
81
82
@classmethod
83
def get_device_capability(cls,
vllm/platforms/hpu.py
+1
@@ -19,6 +19,7 @@ class HpuPlatform(Platform):
19
device_name: str = "hpu"
20
device_type: str = "hpu"
21
dispatch_key: str = "HPU"
22
+ ray_device_key: str = "HPU"
23
24
@classmethod
25
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
vllm/platforms/interface.py
+4
@@ -82,6 +82,10 @@ class Platform:
82
# check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
83
# use "CPU" as a fallback for platforms not registered in PyTorch
84
dispatch_key: str = "CPU"
85
+ # available ray device keys:
86
+ # https://github.com/ray-project/ray/blob/10ba5adadcc49c60af2c358a33bb943fb491a171/python/ray/_private/ray_constants.py#L438 # noqa
87
+ # empty string means the device does not support ray
88
+ ray_device_key: str = ""
89
# The torch.compile backend for compiling simple and
90
# standalone functions. The default value is "inductor" to keep
91
# the same behavior as PyTorch.
vllm/platforms/neuron.py
+1
@@ -16,6 +16,7 @@ class NeuronPlatform(Platform):
16
_enum = PlatformEnum.NEURON
17
device_name: str = "neuron"
18
device_type: str = "neuron"
19
+ ray_device_key: str = "neuron_cores"
20
supported_quantization: list[str] = ["neuron_quant"]
21
22
@classmethod
vllm/platforms/rocm.py
+2
@@ -64,6 +64,8 @@ class RocmPlatform(Platform):
64
device_name: str = "rocm"
65
device_type: str = "cuda"
66
dispatch_key: str = "CUDA"
67
+ ray_device_key: str = "GPU"
68
+
69
supported_quantization: list[str] = [
70
"awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
71
"fbgemm_fp8", "gguf"
vllm/platforms/tpu.py
+2
@@ -19,6 +19,8 @@ class TpuPlatform(Platform):
19
device_name: str = "tpu"
20
device_type: str = "tpu"
21
dispatch_key: str = "XLA"
22
+ ray_device_key: str = "TPU"
23
+
24
supported_quantization: list[str] = [
25
"tpu_int8", "compressed-tensors", "compressed_tensors"
26
]
vllm/platforms/xpu.py
+3
@@ -19,6 +19,9 @@ class XPUPlatform(Platform):
19
device_name: str = "xpu"
20
device_type: str = "xpu"
21
dispatch_key: str = "XPU"
22
+ # Intel XPU's device key is "GPU" for Ray.
23
+ # see https://github.com/ray-project/ray/blob/6a5eb5865eeb9ccf058a79b44f107e327e360673/python/ray/_private/accelerators/intel_gpu.py#L20 # noqa: E501
24
+ ray_device_key: str = "GPU"
25
26
@classmethod
27
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
vllm/v1/executor/ray_utils.py
+11
-2
@@ -41,7 +41,12 @@ try:
41
42
def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
43
node_id = ray.get_runtime_context().get_node_id()
44
- gpu_ids = ray.get_gpu_ids()
44
+ device_key = current_platform.ray_device_key
45
+ if not device_key:
46
+ raise RuntimeError("current platform %s does not support ray.",
47
+ current_platform.device_name)
48
+ gpu_ids = ray.get_runtime_context().get_accelerator_ids(
49
+ )[device_key]
50
return node_id, gpu_ids
51
52
def setup_device_if_necessary(self):
@@ -211,7 +216,11 @@ def initialize_ray_cluster(
216
# Placement group is already set.
217
return
218
214
- device_str = "GPU" if not current_platform.is_tpu() else "TPU"
219
+ device_str = current_platform.ray_device_key
220
+ if not device_str:
221
+ raise ValueError(
222
+ f"current platform {current_platform.device_name} does not "
223
+ "support ray.")
224
# Create placement group for worker processes
225
current_placement_group = ray.util.get_current_placement_group()
226
if current_placement_group: