A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[BugFix] Avoid initializing CUDA too early (#3487)
Nick Hill committed
Mar 18, 2024 at 23:05 UTC
7341c77d693edcecf0a9f5a6e399c5137177dfba
1 file changed
+4
-4
vllm/config.py
+4
-4
@@ -577,12 +577,12 @@ class DeviceConfig:
577
def __init__(self, device: str = "auto") -> None:
578
if device == "auto":
579
# Automated device type detection
580
- if torch.cuda.is_available():
581
- self.device_type = "cuda"
582
- elif is_neuron():
580
+ if is_neuron():
581
self.device_type = "neuron"
582
else:
585
- raise RuntimeError("No supported device detected.")
583
+ # We don't call torch.cuda.is_available() here to
584
+ # avoid initializing CUDA before workers are forked
585
+ self.device_type = "cuda"
586
else:
587
# Device type is assigned explicitly
588
self.device_type = device