@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 89ce62a316
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[platform] add ray_device_key (#11948)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 13, 2025 at 16:20 UTC 89ce62a316e68c50121e74d5a832e0cb9a5101d1
9 files changed +38 -8
vllm/executor/ray_utils.py
+13 -6
@@ -8,6 +8,7 @@ import msgspec
8 from vllm.config import ParallelConfig
9 from vllm.executor.msgspec_utils import decode_hook, encode_hook
10 from vllm.logger import init_logger
11 +from vllm.platforms import current_platform
12 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
13 from vllm.utils import get_ip
14 from vllm.worker.worker_base import WorkerWrapperBase
@@ -47,7 +48,12 @@ try:
48
49 def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
50 node_id = ray.get_runtime_context().get_node_id()
50 - gpu_ids = ray.get_gpu_ids()
51 + device_key = current_platform.ray_device_key
52 + if not device_key:
53 + raise RuntimeError("current platform %s does not support ray.",
54 + current_platform.device_name)
55 + gpu_ids = ray.get_runtime_context().get_accelerator_ids(
56 + )[device_key]
57 return node_id, gpu_ids
58
59 def execute_model_spmd(
@@ -249,11 +255,12 @@ def initialize_ray_cluster(
255 # Placement group is already set.
256 return
257
252 - device_str = "GPU"
253 - if current_platform.is_tpu():
254 - device_str = "TPU"
255 - elif current_platform.is_hpu():
256 - device_str = 'HPU'
258 + device_str = current_platform.ray_device_key
259 + if not device_str:
260 + raise ValueError(
261 + f"current platform {current_platform.device_name} does not "
262 + "support ray.")
263 +
264 # Create placement group for worker processes
265 current_placement_group = ray.util.get_current_placement_group()
266 if current_placement_group:
vllm/platforms/cuda.py
+1
@@ -77,6 +77,7 @@ class CudaPlatformBase(Platform):
77 device_name: str = "cuda"
78 device_type: str = "cuda"
79 dispatch_key: str = "CUDA"
80 + ray_device_key: str = "GPU"
81
82 @classmethod
83 def get_device_capability(cls,
vllm/platforms/hpu.py
+1
@@ -19,6 +19,7 @@ class HpuPlatform(Platform):
19 device_name: str = "hpu"
20 device_type: str = "hpu"
21 dispatch_key: str = "HPU"
22 + ray_device_key: str = "HPU"
23
24 @classmethod
25 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
vllm/platforms/interface.py
+4
@@ -82,6 +82,10 @@ class Platform:
82 # check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
83 # use "CPU" as a fallback for platforms not registered in PyTorch
84 dispatch_key: str = "CPU"
85 + # available ray device keys:
86 + # https://github.com/ray-project/ray/blob/10ba5adadcc49c60af2c358a33bb943fb491a171/python/ray/_private/ray_constants.py#L438 # noqa
87 + # empty string means the device does not support ray
88 + ray_device_key: str = ""
89 # The torch.compile backend for compiling simple and
90 # standalone functions. The default value is "inductor" to keep
91 # the same behavior as PyTorch.
vllm/platforms/neuron.py
+1
@@ -16,6 +16,7 @@ class NeuronPlatform(Platform):
16 _enum = PlatformEnum.NEURON
17 device_name: str = "neuron"
18 device_type: str = "neuron"
19 + ray_device_key: str = "neuron_cores"
20 supported_quantization: list[str] = ["neuron_quant"]
21
22 @classmethod
vllm/platforms/rocm.py
+2
@@ -64,6 +64,8 @@ class RocmPlatform(Platform):
64 device_name: str = "rocm"
65 device_type: str = "cuda"
66 dispatch_key: str = "CUDA"
67 + ray_device_key: str = "GPU"
68 +
69 supported_quantization: list[str] = [
70 "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
71 "fbgemm_fp8", "gguf"
vllm/platforms/tpu.py
+2
@@ -19,6 +19,8 @@ class TpuPlatform(Platform):
19 device_name: str = "tpu"
20 device_type: str = "tpu"
21 dispatch_key: str = "XLA"
22 + ray_device_key: str = "TPU"
23 +
24 supported_quantization: list[str] = [
25 "tpu_int8", "compressed-tensors", "compressed_tensors"
26 ]
vllm/platforms/xpu.py
+3
@@ -19,6 +19,9 @@ class XPUPlatform(Platform):
19 device_name: str = "xpu"
20 device_type: str = "xpu"
21 dispatch_key: str = "XPU"
22 + # Intel XPU's device key is "GPU" for Ray.
23 + # see https://github.com/ray-project/ray/blob/6a5eb5865eeb9ccf058a79b44f107e327e360673/python/ray/_private/accelerators/intel_gpu.py#L20 # noqa: E501
24 + ray_device_key: str = "GPU"
25
26 @classmethod
27 def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
vllm/v1/executor/ray_utils.py
+11 -2
@@ -41,7 +41,12 @@ try:
41
42 def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
43 node_id = ray.get_runtime_context().get_node_id()
44 - gpu_ids = ray.get_gpu_ids()
44 + device_key = current_platform.ray_device_key
45 + if not device_key:
46 + raise RuntimeError("current platform %s does not support ray.",
47 + current_platform.device_name)
48 + gpu_ids = ray.get_runtime_context().get_accelerator_ids(
49 + )[device_key]
50 return node_id, gpu_ids
51
52 def setup_device_if_necessary(self):
@@ -211,7 +216,11 @@ def initialize_ray_cluster(
216 # Placement group is already set.
217 return
218
214 - device_str = "GPU" if not current_platform.is_tpu() else "TPU"
219 + device_str = current_platform.ray_device_key
220 + if not device_str:
221 + raise ValueError(
222 + f"current platform {current_platform.device_name} does not "
223 + "support ray.")
224 # Create placement group for worker processes
225 current_placement_group = ray.util.get_current_placement_group()
226 if current_placement_group: