@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9d827170a3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Platforms] Add `device_type` in `Platform` (#10508)

Signed-off-by: MengqingCao <cmq0113@163.com>

Mengqing Cao committed Nov 21, 2024 at 12:44 UTC 9d827170a3aa586dfb458bf28d18fd279bdbf580
10 files changed +11 -15
vllm/config.py
+2 -15
@@ -1193,21 +1193,8 @@ class DeviceConfig:
1193 def __init__(self, device: str = "auto") -> None:
1194 if device == "auto":
1195 # Automated device type detection
1196 - if current_platform.is_cuda_alike():
1197 - self.device_type = "cuda"
1198 - elif current_platform.is_neuron():
1199 - self.device_type = "neuron"
1200 - elif current_platform.is_hpu():
1201 - self.device_type = "hpu"
1202 - elif current_platform.is_openvino():
1203 - self.device_type = "openvino"
1204 - elif current_platform.is_tpu():
1205 - self.device_type = "tpu"
1206 - elif current_platform.is_cpu():
1207 - self.device_type = "cpu"
1208 - elif current_platform.is_xpu():
1209 - self.device_type = "xpu"
1210 - else:
1196 + self.device_type = current_platform.device_type
1197 + if self.device_type is None:
1198 raise RuntimeError("Failed to infer device type")
1199 else:
1200 # Device type is assigned explicitly
vllm/platforms/cpu.py
+1
@@ -19,6 +19,7 @@ logger = init_logger(__name__)
19
20 class CpuPlatform(Platform):
21 _enum = PlatformEnum.CPU
22 + device_type: str = "cpu"
23
24 @classmethod
25 def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/cuda.py
+1
@@ -109,6 +109,7 @@ def device_id_to_physical_device_id(device_id: int) -> int:
109
110 class CudaPlatform(Platform):
111 _enum = PlatformEnum.CUDA
112 + device_type: str = "cuda"
113
114 @classmethod
115 def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
vllm/platforms/hpu.py
+1
@@ -5,6 +5,7 @@ from .interface import Platform, PlatformEnum, _Backend
5
6 class HpuPlatform(Platform):
7 _enum = PlatformEnum.HPU
8 + device_type: str = "hpu"
9
10 @classmethod
11 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/interface.py
+1
@@ -56,6 +56,7 @@ class DeviceCapability(NamedTuple):
56
57 class Platform:
58 _enum: PlatformEnum
59 + device_type: str
60
61 def is_cuda(self) -> bool:
62 return self._enum == PlatformEnum.CUDA
vllm/platforms/neuron.py
+1
@@ -3,6 +3,7 @@ from .interface import Platform, PlatformEnum
3
4 class NeuronPlatform(Platform):
5 _enum = PlatformEnum.NEURON
6 + device_type: str = "neuron"
7
8 @classmethod
9 def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/openvino.py
+1
@@ -10,6 +10,7 @@ logger = init_logger(__name__)
10
11 class OpenVinoPlatform(Platform):
12 _enum = PlatformEnum.OPENVINO
13 + device_type: str = "openvino"
14
15 @classmethod
16 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/rocm.py
+1
@@ -29,6 +29,7 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
29
30 class RocmPlatform(Platform):
31 _enum = PlatformEnum.ROCM
32 + device_type: str = "cuda"
33
34 @classmethod
35 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/tpu.py
+1
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
16
17 class TpuPlatform(Platform):
18 _enum = PlatformEnum.TPU
19 + device_type: str = "tpu"
20
21 @classmethod
22 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/xpu.py
+1
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
16
17 class XPUPlatform(Platform):
18 _enum = PlatformEnum.XPU
19 + device_type: str = "xpu"
20
21 @classmethod
22 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend: