A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Misc] Increase RayDistributedExecutor RAY_CGRAPH_get_timeout (#15301)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
Rui Qiao committed
Mar 21, 2025 at 22:25 UTC
4c69e228b32220ac9159dfdcf0df13ea776e630d
1 file changed
+9
vllm/executor/ray_distributed_executor.py
+9
@@ -561,6 +561,15 @@ class RayDistributedExecutor(DistributedExecutorBase):
561
envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL)
562
logger.info("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM = %s",
563
envs.VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM)
564
+ # Enlarge the default value of "RAY_CGRAPH_get_timeout" to 300 seconds
565
+ # (it is 10 seconds by default). This is a Ray environment variable to
566
+ # control the timeout of getting result from a compiled graph execution,
567
+ # i.e., the distributed execution that includes model forward runs and
568
+ # intermediate tensor communications, in the case of vllm.
569
+ os.environ.setdefault("RAY_CGRAPH_get_timeout", "300") # noqa: SIM112
570
+ logger.info("RAY_CGRAPH_get_timeout is set to %s",
571
+ os.environ["RAY_CGRAPH_get_timeout"]) # noqa: SIM112
572
+
573
with InputNode() as input_data:
574
# Example DAG: PP=2, TP=4
575
#