@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1c0afa13c5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[BugFix] Don't start a Ray cluster when not using Ray (#5570)

Nick Hill committed Jun 15, 2024 at 16:30 UTC 1c0afa13c57766641e75172ff1cac2e09f79a3b9
1 file changed +7 -2
vllm/config.py
+7 -2
@@ -616,9 +616,14 @@ class ParallelConfig:
616 "required for multi-node inference")
617 backend = "ray"
618 elif ray_found:
619 - from ray.util import get_current_placement_group
620 - if self.placement_group or get_current_placement_group():
619 + if self.placement_group:
620 backend = "ray"
621 + else:
622 + from ray import is_initialized as ray_is_initialized
623 + if ray_is_initialized():
624 + from ray.util import get_current_placement_group
625 + if get_current_placement_group():
626 + backend = "ray"
627 self.distributed_executor_backend = backend
628 logger.info("Defaulting to use %s for distributed inference",
629 backend)