A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[MISC] Make GroupCoordinator compatible with out-of-tree devices (#16464)
Signed-off-by: hzji210@gmail.com <hzji210@gmail.com>
Huazhong Ji committed
Apr 12, 2025 at 17:20 UTC
68bb122eb458a42169dd9d1772a7b32b84b2be95
1 file changed
+3
-1
vllm/distributed/parallel_state.py
+3
-1
@@ -194,9 +194,11 @@ class GroupCoordinator:
194
195
from vllm.platforms import current_platform
196
197
- # TODO: fix it for other platforms
197
if current_platform.is_cuda_alike():
198
self.device = torch.device(f"cuda:{local_rank}")
199
+ elif current_platform.is_out_of_tree():
200
+ self.device = torch.device(
201
+ f"{current_platform.device_name}:{local_rank}")
202
else:
203
self.device = torch.device("cpu")
204