@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7341c77d69
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[BugFix] Avoid initializing CUDA too early (#3487)

Nick Hill committed Mar 18, 2024 at 23:05 UTC 7341c77d693edcecf0a9f5a6e399c5137177dfba
1 file changed +4 -4
vllm/config.py
+4 -4
@@ -577,12 +577,12 @@ class DeviceConfig:
577 def __init__(self, device: str = "auto") -> None:
578 if device == "auto":
579 # Automated device type detection
580 - if torch.cuda.is_available():
581 - self.device_type = "cuda"
582 - elif is_neuron():
580 + if is_neuron():
581 self.device_type = "neuron"
582 else:
585 - raise RuntimeError("No supported device detected.")
583 + # We don't call torch.cuda.is_available() here to
584 + # avoid initializing CUDA before workers are forked
585 + self.device_type = "cuda"
586 else:
587 # Device type is assigned explicitly
588 self.device_type = device