@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fbf2564554
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] Add `RayExecutor` support for `AsyncLLM` (api server) (#11712)

Kunshang Ji committed Jan 4, 2025 at 14:41 UTC fbf25645542fdcfb3f1a27ba05486492e368925c
1 file changed +6 -1
vllm/v1/engine/async_llm.py
+6 -1
@@ -22,6 +22,7 @@ from vllm.v1.engine.core_client import EngineCoreClient
22 from vllm.v1.engine.detokenizer import Detokenizer
23 from vllm.v1.engine.processor import Processor
24 from vllm.v1.executor.abstract import Executor
25 +from vllm.v1.executor.ray_utils import initialize_ray_cluster
26
27 logger = init_logger(__name__)
28
@@ -131,7 +132,11 @@ class AsyncLLM(EngineClient):
132 executor_class: Type[Executor]
133 distributed_executor_backend = (
134 vllm_config.parallel_config.distributed_executor_backend)
134 - if distributed_executor_backend == "mp":
135 + if distributed_executor_backend == "ray":
136 + initialize_ray_cluster(vllm_config.parallel_config)
137 + from vllm.v1.executor.ray_executor import RayExecutor
138 + executor_class = RayExecutor
139 + elif distributed_executor_backend == "mp":
140 from vllm.v1.executor.multiproc_executor import MultiprocExecutor
141 executor_class = MultiprocExecutor
142 else: