@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0d8451c3a4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Distributed] Allow the placement group more time to wait for resources to be ready (#11138)

Signed-off-by: Jiaxin Shan <seedjeffwan@gmail.com>

Jiaxin Shan committed Dec 13, 2024 at 12:17 UTC 0d8451c3a45d309e58de5e1c546f043de461d478
1 file changed +7 -3
vllm/executor/ray_utils.py
+7 -3
@@ -277,10 +277,14 @@ def initialize_ray_cluster(
277 f"Total number of devices: {device_bundles}.")
278 else:
279 num_devices_in_cluster = ray.cluster_resources().get(device_str, 0)
280 + # Log a warning message and delay resource allocation failure response.
281 + # Avoid immediate rejection to allow user-initiated placement group
282 + # created and wait cluster to be ready
283 if parallel_config.world_size > num_devices_in_cluster:
281 - raise ValueError(
282 - f"The number of required {device_str}s exceeds the total "
283 - f"number of available {device_str}s in the placement group.")
284 + logger.warning(
285 + "The number of required %ss exceeds the total "
286 + "number of available %ss in the placement group.", device_str,
287 + device_str)
288 # Create a new placement group
289 placement_group_specs: List[Dict[str, float]] = ([{
290 device_str: 1.0