A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
Fix the AMD weight loading tests (#8390)
Michael Goin committed
Sep 11, 2024 at 23:35 UTC
b6c75e1cf27681ec92629930c03b616c7c9b9929
1 file changed
+1
-1
vllm/config.py
+1
-1
@@ -883,7 +883,7 @@ class ParallelConfig:
883
from vllm.executor import ray_utils
884
backend = "mp"
885
ray_found = ray_utils.ray_is_available()
886
- if (torch.cuda.is_available()
886
+ if (current_platform.is_cuda()
887
and cuda_device_count_stateless() < self.world_size):
888
if not ray_found:
889
raise ValueError("Unable to load Ray which is "