@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f218f9c24d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[core] Turn off GPU communication overlap for Ray executor (#12051)

Signed-off-by: Rui Qiao <ruisearch42@gmail.com>

Rui Qiao committed Jan 14, 2025 at 21:19 UTC f218f9c24d224800e0ea4488aa71bd8215c8bdcd
1 file changed +4 -4
vllm/envs.py
+4 -4
@@ -45,7 +45,7 @@ if TYPE_CHECKING:
45 VLLM_USE_RAY_SPMD_WORKER: bool = False
46 VLLM_USE_RAY_COMPILED_DAG: bool = False
47 VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL: bool = True
48 - VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = True
48 + VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False
49 VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
50 VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
51 VLLM_IMAGE_FETCH_TIMEOUT: int = 5
@@ -340,11 +340,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
340 lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL", "1"))
341 ),
342
343 - # If the env var is set, it enables GPU communication overlap in
344 - # Ray's compiled DAG. This flag is ignored if
343 + # If the env var is set, it enables GPU communication overlap
344 + # (experimental feature) in Ray's compiled DAG. This flag is ignored if
345 # VLLM_USE_RAY_COMPILED_DAG is not set.
346 "VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM":
347 - lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM", "1"))
347 + lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM", "0"))
348 ),
349
350 # Use dedicated multiprocess context for workers.