A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Platforms] Add `device_type` in `Platform` (#10508)
Signed-off-by: MengqingCao <cmq0113@163.com>
Mengqing Cao committed
Nov 21, 2024 at 12:44 UTC
9d827170a3aa586dfb458bf28d18fd279bdbf580
10 files changed
+11
-15
vllm/config.py
+2
-15
@@ -1193,21 +1193,8 @@ class DeviceConfig:
1193
def __init__(self, device: str = "auto") -> None:
1194
if device == "auto":
1195
# Automated device type detection
1196
- if current_platform.is_cuda_alike():
1197
- self.device_type = "cuda"
1198
- elif current_platform.is_neuron():
1199
- self.device_type = "neuron"
1200
- elif current_platform.is_hpu():
1201
- self.device_type = "hpu"
1202
- elif current_platform.is_openvino():
1203
- self.device_type = "openvino"
1204
- elif current_platform.is_tpu():
1205
- self.device_type = "tpu"
1206
- elif current_platform.is_cpu():
1207
- self.device_type = "cpu"
1208
- elif current_platform.is_xpu():
1209
- self.device_type = "xpu"
1210
- else:
1196
+ self.device_type = current_platform.device_type
1197
+ if self.device_type is None:
1198
raise RuntimeError("Failed to infer device type")
1199
else:
1200
# Device type is assigned explicitly
vllm/platforms/cpu.py
+1
@@ -19,6 +19,7 @@ logger = init_logger(__name__)
19
20
class CpuPlatform(Platform):
21
_enum = PlatformEnum.CPU
22
+ device_type: str = "cpu"
23
24
@classmethod
25
def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/cuda.py
+1
@@ -109,6 +109,7 @@ def device_id_to_physical_device_id(device_id: int) -> int:
109
110
class CudaPlatform(Platform):
111
_enum = PlatformEnum.CUDA
112
+ device_type: str = "cuda"
113
114
@classmethod
115
def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
vllm/platforms/hpu.py
+1
@@ -5,6 +5,7 @@ from .interface import Platform, PlatformEnum, _Backend
5
6
class HpuPlatform(Platform):
7
_enum = PlatformEnum.HPU
8
+ device_type: str = "hpu"
9
10
@classmethod
11
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/interface.py
+1
@@ -56,6 +56,7 @@ class DeviceCapability(NamedTuple):
56
57
class Platform:
58
_enum: PlatformEnum
59
+ device_type: str
60
61
def is_cuda(self) -> bool:
62
return self._enum == PlatformEnum.CUDA
vllm/platforms/neuron.py
+1
@@ -3,6 +3,7 @@ from .interface import Platform, PlatformEnum
3
4
class NeuronPlatform(Platform):
5
_enum = PlatformEnum.NEURON
6
+ device_type: str = "neuron"
7
8
@classmethod
9
def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/openvino.py
+1
@@ -10,6 +10,7 @@ logger = init_logger(__name__)
10
11
class OpenVinoPlatform(Platform):
12
_enum = PlatformEnum.OPENVINO
13
+ device_type: str = "openvino"
14
15
@classmethod
16
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/rocm.py
+1
@@ -29,6 +29,7 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
29
30
class RocmPlatform(Platform):
31
_enum = PlatformEnum.ROCM
32
+ device_type: str = "cuda"
33
34
@classmethod
35
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/tpu.py
+1
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
16
17
class TpuPlatform(Platform):
18
_enum = PlatformEnum.TPU
19
+ device_type: str = "tpu"
20
21
@classmethod
22
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/xpu.py
+1
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
16
17
class XPUPlatform(Platform):
18
_enum = PlatformEnum.XPU
19
+ device_type: str = "xpu"
20
21
@classmethod
22
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend: