A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] Add `RayExecutor` support for `AsyncLLM` (api server) (#11712)
Kunshang Ji committed
Jan 4, 2025 at 14:41 UTC
fbf25645542fdcfb3f1a27ba05486492e368925c
1 file changed
+6
-1
vllm/v1/engine/async_llm.py
+6
-1
@@ -22,6 +22,7 @@ from vllm.v1.engine.core_client import EngineCoreClient
22
from vllm.v1.engine.detokenizer import Detokenizer
23
from vllm.v1.engine.processor import Processor
24
from vllm.v1.executor.abstract import Executor
25
+from vllm.v1.executor.ray_utils import initialize_ray_cluster
26
27
logger = init_logger(__name__)
28
@@ -131,7 +132,11 @@ class AsyncLLM(EngineClient):
132
executor_class: Type[Executor]
133
distributed_executor_backend = (
134
vllm_config.parallel_config.distributed_executor_backend)
134
- if distributed_executor_backend == "mp":
135
+ if distributed_executor_backend == "ray":
136
+ initialize_ray_cluster(vllm_config.parallel_config)
137
+ from vllm.v1.executor.ray_executor import RayExecutor
138
+ executor_class = RayExecutor
139
+ elif distributed_executor_backend == "mp":
140
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
141
executor_class = MultiprocExecutor
142
else: