A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[BugFix] Don't start a Ray cluster when not using Ray (#5570)
Nick Hill committed
Jun 15, 2024 at 16:30 UTC
1c0afa13c57766641e75172ff1cac2e09f79a3b9
1 file changed
+7
-2
vllm/config.py
+7
-2
@@ -616,9 +616,14 @@ class ParallelConfig:
616
"required for multi-node inference")
617
backend = "ray"
618
elif ray_found:
619
- from ray.util import get_current_placement_group
620
- if self.placement_group or get_current_placement_group():
619
+ if self.placement_group:
620
backend = "ray"
621
+ else:
622
+ from ray import is_initialized as ray_is_initialized
623
+ if ray_is_initialized():
624
+ from ray.util import get_current_placement_group
625
+ if get_current_placement_group():
626
+ backend = "ray"
627
self.distributed_executor_backend = backend
628
logger.info("Defaulting to use %s for distributed inference",
629
backend)