A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[core] Turn off GPU communication overlap for Ray executor (#12051)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
Rui Qiao committed
Jan 14, 2025 at 21:19 UTC
f218f9c24d224800e0ea4488aa71bd8215c8bdcd
1 file changed
+4
-4
vllm/envs.py
+4
-4
@@ -45,7 +45,7 @@ if TYPE_CHECKING:
45
VLLM_USE_RAY_SPMD_WORKER: bool = False
46
VLLM_USE_RAY_COMPILED_DAG: bool = False
47
VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL: bool = True
48
- VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = True
48
+ VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False
49
VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
50
VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
51
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
@@ -340,11 +340,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
340
lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL", "1"))
341
),
342
343
- # If the env var is set, it enables GPU communication overlap in
344
- # Ray's compiled DAG. This flag is ignored if
343
+ # If the env var is set, it enables GPU communication overlap
344
+ # (experimental feature) in Ray's compiled DAG. This flag is ignored if
345
# VLLM_USE_RAY_COMPILED_DAG is not set.
346
"VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM":
347
- lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM", "1"))
347
+ lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM", "0"))
348
),
349
350
# Use dedicated multiprocess context for workers.