A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[platforms] improve error message for unspecified platforms (#10520)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 20, 2024 at 23:07 UTC
aaddce5d268d2c82d49b0240d6c112ba4941f69e
2 files changed
+3
-1
vllm/config.py
+2
-1
@@ -1191,12 +1191,13 @@ class SchedulerConfig:
1191
1192
class DeviceConfig:
1193
device: Optional[torch.device]
1194
+ device_type: str
1195
1196
def __init__(self, device: str = "auto") -> None:
1197
if device == "auto":
1198
# Automated device type detection
1199
self.device_type = current_platform.device_type
1199
- if self.device_type is None:
1200
+ if not self.device_type:
1201
raise RuntimeError("Failed to infer device type")
1202
else:
1203
# Device type is assigned explicitly
vllm/platforms/interface.py
+1
@@ -170,3 +170,4 @@ class Platform:
170
171
class UnspecifiedPlatform(Platform):
172
_enum = PlatformEnum.UNSPECIFIED
173
+ device_type = ""