A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 19:00 UTC
platforms: add uses_cpu_device() hook to Platform for DeviceConfig (#42313)
Signed-off-by: Viktor Pus <viktorpus@tenstorrent.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Viktor Pus committed
May 12, 2026 at 21:39 UTC
0ce6613b9ca126ce62943a666b9e6ebd71047cd3
2 files changed
+11
-2
vllm/config/device.py
+7
-2
@@ -65,8 +65,13 @@ class DeviceConfig:
65
elif isinstance(self.device, torch.device):
66
self.device_type = self.device.type
67
68
- # Some device types require processing inputs on CPU
69
- if self.device_type in ["tpu"]:
68
+ # Some platforms require processing inputs on CPU.
69
+ from vllm.platforms import current_platform
70
+
71
+ if (
72
+ current_platform.uses_host_device_handling()
73
+ and self.device_type == current_platform.device_type
74
+ ):
75
self.device = None
76
else:
77
# Set device with device type
vllm/platforms/interface.py
+4
@@ -172,6 +172,10 @@ class Platform:
172
def is_cpu(self) -> bool:
173
return self._enum == PlatformEnum.CPU
174
175
+ def uses_host_device_handling(self) -> bool:
176
+ """Whether vLLM should leave DeviceConfig.device unset."""
177
+ return self.is_tpu()
178
+
179
def is_zen_cpu(self) -> bool:
180
return False
181