A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] Refactor get_executor_cls (#11754)
Rui Qiao committed
Jan 5, 2025 at 23:59 UTC
022c5c6944bcf28ac4d0d28ce14f2b559358be52
5 files changed
+26
-46
tests/v1/engine/test_engine_core.py
+3
-3
@@ -8,8 +8,8 @@ from vllm import SamplingParams
8
from vllm.engine.arg_utils import EngineArgs
9
from vllm.platforms import current_platform
10
from vllm.v1.engine import EngineCoreRequest
11
-from vllm.v1.engine.async_llm import AsyncLLM
11
from vllm.v1.engine.core import EngineCore
12
+from vllm.v1.executor.abstract import Executor
13
14
if not current_platform.is_cuda():
15
pytest.skip(reason="V1 currently only supported on CUDA.",
@@ -43,7 +43,7 @@ def test_engine_core(monkeypatch):
43
"""Setup the EngineCore."""
44
engine_args = EngineArgs(model=MODEL_NAME)
45
vllm_config = engine_args.create_engine_config()
46
- executor_class = AsyncLLM._get_executor_cls(vllm_config)
46
+ executor_class = Executor.get_class(vllm_config)
47
48
engine_core = EngineCore(vllm_config=vllm_config,
49
executor_class=executor_class)
@@ -149,7 +149,7 @@ def test_engine_core_advanced_sampling(monkeypatch):
149
"""Setup the EngineCore."""
150
engine_args = EngineArgs(model=MODEL_NAME)
151
vllm_config = engine_args.create_engine_config()
152
- executor_class = AsyncLLM._get_executor_cls(vllm_config)
152
+ executor_class = Executor.get_class(vllm_config)
153
154
engine_core = EngineCore(vllm_config=vllm_config,
155
executor_class=executor_class)
tests/v1/engine/test_engine_core_client.py
+3
-3
@@ -11,8 +11,8 @@ from vllm.engine.arg_utils import EngineArgs
11
from vllm.platforms import current_platform
12
from vllm.usage.usage_lib import UsageContext
13
from vllm.v1.engine import EngineCoreRequest
14
-from vllm.v1.engine.async_llm import AsyncLLM
14
from vllm.v1.engine.core_client import EngineCoreClient
15
+from vllm.v1.executor.abstract import Executor
16
17
if not current_platform.is_cuda():
18
pytest.skip(reason="V1 currently only supported on CUDA.",
@@ -84,7 +84,7 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
84
engine_args = EngineArgs(model=MODEL_NAME, compilation_config=3)
85
vllm_config = engine_args.create_engine_config(
86
UsageContext.UNKNOWN_CONTEXT)
87
- executor_class = AsyncLLM._get_executor_cls(vllm_config)
87
+ executor_class = Executor.get_class(vllm_config)
88
client = EngineCoreClient.make_client(
89
multiprocess_mode=multiprocessing_mode,
90
asyncio_mode=False,
@@ -152,7 +152,7 @@ async def test_engine_core_client_asyncio(monkeypatch):
152
engine_args = EngineArgs(model=MODEL_NAME)
153
vllm_config = engine_args.create_engine_config(
154
usage_context=UsageContext.UNKNOWN_CONTEXT)
155
- executor_class = AsyncLLM._get_executor_cls(vllm_config)
155
+ executor_class = Executor.get_class(vllm_config)
156
client = EngineCoreClient.make_client(
157
multiprocess_mode=True,
158
asyncio_mode=True,
vllm/v1/engine/async_llm.py
+1
-20
@@ -22,7 +22,6 @@ from vllm.v1.engine.core_client import EngineCoreClient
22
from vllm.v1.engine.detokenizer import Detokenizer
23
from vllm.v1.engine.processor import Processor
24
from vllm.v1.executor.abstract import Executor
25
-from vllm.v1.executor.ray_utils import initialize_ray_cluster
25
26
logger = init_logger(__name__)
27
@@ -105,7 +104,7 @@ class AsyncLLM(EngineClient):
104
else:
105
vllm_config = engine_config
106
108
- executor_class = cls._get_executor_cls(vllm_config)
107
+ executor_class = Executor.get_class(vllm_config)
108
109
# Create the AsyncLLM.
110
return cls(
@@ -127,24 +126,6 @@ class AsyncLLM(EngineClient):
126
if handler := getattr(self, "output_handler", None):
127
handler.cancel()
128
130
- @classmethod
131
- def _get_executor_cls(cls, vllm_config: VllmConfig) -> Type[Executor]:
132
- executor_class: Type[Executor]
133
- distributed_executor_backend = (
134
- vllm_config.parallel_config.distributed_executor_backend)
135
- if distributed_executor_backend == "ray":
136
- initialize_ray_cluster(vllm_config.parallel_config)
137
- from vllm.v1.executor.ray_executor import RayExecutor
138
- executor_class = RayExecutor
139
- elif distributed_executor_backend == "mp":
140
- from vllm.v1.executor.multiproc_executor import MultiprocExecutor
141
- executor_class = MultiprocExecutor
142
- else:
143
- assert (distributed_executor_backend is None)
144
- from vllm.v1.executor.uniproc_executor import UniprocExecutor
145
- executor_class = UniprocExecutor
146
- return executor_class
147
-
129
async def add_request(
130
self,
131
request_id: str,
vllm/v1/engine/llm_engine.py
+1
-19
@@ -89,7 +89,7 @@ class LLMEngine:
89
90
# Create the engine configs.
91
vllm_config = engine_args.create_engine_config(usage_context)
92
- executor_class = cls._get_executor_cls(vllm_config)
92
+ executor_class = Executor.get_class(vllm_config)
93
94
if VLLM_ENABLE_V1_MULTIPROCESSING:
95
logger.debug("Enabling multiprocessing for LLMEngine.")
@@ -103,24 +103,6 @@ class LLMEngine:
103
stat_loggers=stat_loggers,
104
multiprocess_mode=enable_multiprocessing)
105
106
- @classmethod
107
- def _get_executor_cls(cls, vllm_config: VllmConfig) -> Type[Executor]:
108
- executor_class: Type[Executor]
109
- distributed_executor_backend = (
110
- vllm_config.parallel_config.distributed_executor_backend)
111
- if distributed_executor_backend == "ray":
112
- from vllm.v1.executor.ray_executor import RayExecutor
113
- executor_class = RayExecutor
114
- elif distributed_executor_backend == "mp":
115
- from vllm.v1.executor.multiproc_executor import MultiprocExecutor
116
- executor_class = MultiprocExecutor
117
- else:
118
- assert (distributed_executor_backend is None)
119
- from vllm.v1.executor.uniproc_executor import UniprocExecutor
120
- executor_class = UniprocExecutor
121
-
122
- return executor_class
123
-
106
def get_num_unfinished_requests(self) -> int:
107
return self.detokenizer.get_num_unfinished_requests()
108
vllm/v1/executor/abstract.py
+18
-1
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Tuple
2
+from typing import Tuple, Type
3
4
from vllm.config import VllmConfig
5
from vllm.v1.outputs import ModelRunnerOutput
@@ -8,6 +8,23 @@ from vllm.v1.outputs import ModelRunnerOutput
8
class Executor(ABC):
9
"""Abstract class for executors."""
10
11
+ @staticmethod
12
+ def get_class(vllm_config: VllmConfig) -> Type["Executor"]:
13
+ executor_class: Type[Executor]
14
+ distributed_executor_backend = (
15
+ vllm_config.parallel_config.distributed_executor_backend)
16
+ if distributed_executor_backend == "ray":
17
+ from vllm.v1.executor.ray_executor import RayExecutor
18
+ executor_class = RayExecutor
19
+ elif distributed_executor_backend == "mp":
20
+ from vllm.v1.executor.multiproc_executor import MultiprocExecutor
21
+ executor_class = MultiprocExecutor
22
+ else:
23
+ assert (distributed_executor_backend is None)
24
+ from vllm.v1.executor.uniproc_executor import UniprocExecutor
25
+ executor_class = UniprocExecutor
26
+ return executor_class
27
+
28
@abstractmethod
29
def __init__(self, vllm_config: VllmConfig) -> None:
30
raise NotImplementedError