@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9832e5572a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Unify VLLM_ENABLE_V1_MULTIPROCESSING handling in RayExecutor (#11472)

Rui Qiao committed Dec 24, 2024 at 19:49 UTC 9832e5572a602967beac0ccb8a4eb65bc18478a3
3 files changed +4 -8
tests/basic_correctness/test_basic_correctness.py
-5
@@ -127,11 +127,6 @@ def test_models_distributed(
127 if attention_backend:
128 os.environ["VLLM_ATTENTION_BACKEND"] = attention_backend
129
130 - # Import VLLM_USE_V1 dynamically to handle patching
131 - from vllm.envs import VLLM_USE_V1
132 - if VLLM_USE_V1 and distributed_executor_backend != "mp":
133 - os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
134 -
130 dtype = "half"
131 max_tokens = 5
132
vllm/v1/engine/llm_engine.py
-2
@@ -21,7 +21,6 @@ from vllm.v1.engine.core_client import EngineCoreClient
21 from vllm.v1.engine.detokenizer import Detokenizer
22 from vllm.v1.engine.processor import Processor
23 from vllm.v1.executor.abstract import Executor
24 -from vllm.v1.executor.ray_utils import initialize_ray_cluster
24
25 logger = init_logger(__name__)
26
@@ -112,7 +111,6 @@ class LLMEngine:
111 distributed_executor_backend = (
112 vllm_config.parallel_config.distributed_executor_backend)
113 if distributed_executor_backend == "ray":
115 - initialize_ray_cluster(vllm_config.parallel_config)
114 from vllm.v1.executor.ray_executor import RayExecutor
115 executor_class = RayExecutor
116 elif distributed_executor_backend == "mp":
vllm/v1/executor/ray_executor.py
+4 -1
@@ -8,7 +8,8 @@ from vllm.config import VllmConfig
8 from vllm.logger import init_logger
9 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10 from vllm.v1.executor.abstract import Executor
11 -from vllm.v1.executor.ray_utils import RayWorkerWrapper, ray
11 +from vllm.v1.executor.ray_utils import (RayWorkerWrapper,
12 + initialize_ray_cluster, ray)
13 from vllm.v1.outputs import ModelRunnerOutput
14
15 if ray is not None:
@@ -33,7 +34,9 @@ class RayExecutor(Executor):
34 if ray_usage != "1":
35 os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
36
37 + initialize_ray_cluster(self.parallel_config)
38 placement_group = self.parallel_config.placement_group
39 +
40 # Create the parallel GPU workers.
41 self._init_workers_ray(placement_group)
42