@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aaddce5d26
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[platforms] improve error message for unspecified platforms (#10520)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 20, 2024 at 23:07 UTC aaddce5d268d2c82d49b0240d6c112ba4941f69e
2 files changed +3 -1
vllm/config.py
+2 -1
@@ -1191,12 +1191,13 @@ class SchedulerConfig:
1191
1192 class DeviceConfig:
1193 device: Optional[torch.device]
1194 + device_type: str
1195
1196 def __init__(self, device: str = "auto") -> None:
1197 if device == "auto":
1198 # Automated device type detection
1199 self.device_type = current_platform.device_type
1199 - if self.device_type is None:
1200 + if not self.device_type:
1201 raise RuntimeError("Failed to infer device type")
1202 else:
1203 # Device type is assigned explicitly
vllm/platforms/interface.py
+1
@@ -170,3 +170,4 @@ class Platform:
170
171 class UnspecifiedPlatform(Platform):
172 _enum = PlatformEnum.UNSPECIFIED
173 + device_type = ""