A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Unify VLLM_ENABLE_V1_MULTIPROCESSING handling in RayExecutor (#11472)
Rui Qiao committed
Dec 24, 2024 at 19:49 UTC
9832e5572a602967beac0ccb8a4eb65bc18478a3
3 files changed
+4
-8
tests/basic_correctness/test_basic_correctness.py
-5
@@ -127,11 +127,6 @@ def test_models_distributed(
127
if attention_backend:
128
os.environ["VLLM_ATTENTION_BACKEND"] = attention_backend
129
130
- # Import VLLM_USE_V1 dynamically to handle patching
131
- from vllm.envs import VLLM_USE_V1
132
- if VLLM_USE_V1 and distributed_executor_backend != "mp":
133
- os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
134
-
130
dtype = "half"
131
max_tokens = 5
132
vllm/v1/engine/llm_engine.py
-2
@@ -21,7 +21,6 @@ from vllm.v1.engine.core_client import EngineCoreClient
21
from vllm.v1.engine.detokenizer import Detokenizer
22
from vllm.v1.engine.processor import Processor
23
from vllm.v1.executor.abstract import Executor
24
-from vllm.v1.executor.ray_utils import initialize_ray_cluster
24
25
logger = init_logger(__name__)
26
@@ -112,7 +111,6 @@ class LLMEngine:
111
distributed_executor_backend = (
112
vllm_config.parallel_config.distributed_executor_backend)
113
if distributed_executor_backend == "ray":
115
- initialize_ray_cluster(vllm_config.parallel_config)
114
from vllm.v1.executor.ray_executor import RayExecutor
115
executor_class = RayExecutor
116
elif distributed_executor_backend == "mp":
vllm/v1/executor/ray_executor.py
+4
-1
@@ -8,7 +8,8 @@ from vllm.config import VllmConfig
8
from vllm.logger import init_logger
9
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10
from vllm.v1.executor.abstract import Executor
11
-from vllm.v1.executor.ray_utils import RayWorkerWrapper, ray
11
+from vllm.v1.executor.ray_utils import (RayWorkerWrapper,
12
+ initialize_ray_cluster, ray)
13
from vllm.v1.outputs import ModelRunnerOutput
14
15
if ray is not None:
@@ -33,7 +34,9 @@ class RayExecutor(Executor):
34
if ray_usage != "1":
35
os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
36
37
+ initialize_ray_cluster(self.parallel_config)
38
placement_group = self.parallel_config.placement_group
39
+
40
# Create the parallel GPU workers.
41
self._init_workers_ray(placement_group)
42