A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[MISC] correct DeviceConfig device field static type analysis (#19699)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Jun 18, 2025 at 08:21 UTC
6e9cc73f674d913113cc1fd5d50f53a2ee94aa8e
2 files changed
+7
-3
vllm/config.py
+5
-2
@@ -2285,7 +2285,7 @@ Device = Literal["auto", "cuda", "neuron", "cpu", "tpu", "xpu", "hpu"]
2285
class DeviceConfig:
2286
"""Configuration for the device to use for vLLM execution."""
2287
2288
- device: SkipValidation[Union[Device, torch.device]] = "auto"
2288
+ device: SkipValidation[Optional[Union[Device, torch.device]]] = "auto"
2289
"""Device type for vLLM execution.
2290
This parameter is deprecated and will be
2291
removed in a future release.
@@ -2327,7 +2327,10 @@ class DeviceConfig:
2327
"to turn on verbose logging to help debug the issue.")
2328
else:
2329
# Device type is assigned explicitly
2330
- self.device_type = self.device
2330
+ if isinstance(self.device, str):
2331
+ self.device_type = self.device
2332
+ elif isinstance(self.device, torch.device):
2333
+ self.device_type = self.device.type
2334
2335
# Some device types require processing inputs on CPU
2336
if self.device_type in ["neuron"]:
vllm/engine/arg_utils.py
+2
-1
@@ -1018,7 +1018,8 @@ class EngineArgs:
1018
from vllm.platforms import current_platform
1019
current_platform.pre_register_and_update()
1020
1021
- device_config = DeviceConfig(device=current_platform.device_type)
1021
+ device_config = DeviceConfig(
1022
+ device=cast(Device, current_platform.device_type))
1023
model_config = self.create_model_config()
1024
1025
# * If VLLM_USE_V1 is unset, we enable V1 for "supported features"