A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Distributed] Allow the placement group more time to wait for resources to be ready (#11138)
Signed-off-by: Jiaxin Shan <seedjeffwan@gmail.com>
Jiaxin Shan committed
Dec 13, 2024 at 12:17 UTC
0d8451c3a45d309e58de5e1c546f043de461d478
1 file changed
+7
-3
vllm/executor/ray_utils.py
+7
-3
@@ -277,10 +277,14 @@ def initialize_ray_cluster(
277
f"Total number of devices: {device_bundles}.")
278
else:
279
num_devices_in_cluster = ray.cluster_resources().get(device_str, 0)
280
+ # Log a warning message and delay resource allocation failure response.
281
+ # Avoid immediate rejection to allow user-initiated placement group
282
+ # created and wait cluster to be ready
283
if parallel_config.world_size > num_devices_in_cluster:
281
- raise ValueError(
282
- f"The number of required {device_str}s exceeds the total "
283
- f"number of available {device_str}s in the placement group.")
284
+ logger.warning(
285
+ "The number of required %ss exceeds the total "
286
+ "number of available %ss in the placement group.", device_str,
287
+ device_str)
288
# Create a new placement group
289
placement_group_specs: List[Dict[str, float]] = ([{
290
device_str: 1.0