A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix ray instance detect issue (#9439)
Yan Ma committed
Oct 28, 2024 at 15:13 UTC
2adb4409e0359039135b5aa6501994da12aa5a26
1 file changed
+10
-3
vllm/executor/ray_utils.py
+10
-3
@@ -232,9 +232,16 @@ def initialize_ray_cluster(
232
233
# Connect to a ray cluster.
234
if current_platform.is_rocm() or current_platform.is_xpu():
235
- ray.init(address=ray_address,
236
- ignore_reinit_error=True,
237
- num_gpus=parallel_config.world_size)
235
+ # Try to connect existing ray instance and create a new one if not found
236
+ try:
237
+ ray.init("auto")
238
+ except ConnectionError:
239
+ logger.warning(
240
+ "No existing RAY instance detected. "
241
+ "A new instance will be launched with current node resources.")
242
+ ray.init(address=ray_address,
243
+ ignore_reinit_error=True,
244
+ num_gpus=parallel_config.world_size)
245
else:
246
ray.init(address=ray_address, ignore_reinit_error=True)
247