@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 022c5c6944
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] Refactor get_executor_cls (#11754)

Rui Qiao committed Jan 5, 2025 at 23:59 UTC 022c5c6944bcf28ac4d0d28ce14f2b559358be52
5 files changed +26 -46
tests/v1/engine/test_engine_core.py
+3 -3
@@ -8,8 +8,8 @@ from vllm import SamplingParams
8 from vllm.engine.arg_utils import EngineArgs
9 from vllm.platforms import current_platform
10 from vllm.v1.engine import EngineCoreRequest
11 -from vllm.v1.engine.async_llm import AsyncLLM
11 from vllm.v1.engine.core import EngineCore
12 +from vllm.v1.executor.abstract import Executor
13
14 if not current_platform.is_cuda():
15 pytest.skip(reason="V1 currently only supported on CUDA.",
@@ -43,7 +43,7 @@ def test_engine_core(monkeypatch):
43 """Setup the EngineCore."""
44 engine_args = EngineArgs(model=MODEL_NAME)
45 vllm_config = engine_args.create_engine_config()
46 - executor_class = AsyncLLM._get_executor_cls(vllm_config)
46 + executor_class = Executor.get_class(vllm_config)
47
48 engine_core = EngineCore(vllm_config=vllm_config,
49 executor_class=executor_class)
@@ -149,7 +149,7 @@ def test_engine_core_advanced_sampling(monkeypatch):
149 """Setup the EngineCore."""
150 engine_args = EngineArgs(model=MODEL_NAME)
151 vllm_config = engine_args.create_engine_config()
152 - executor_class = AsyncLLM._get_executor_cls(vllm_config)
152 + executor_class = Executor.get_class(vllm_config)
153
154 engine_core = EngineCore(vllm_config=vllm_config,
155 executor_class=executor_class)
tests/v1/engine/test_engine_core_client.py
+3 -3
@@ -11,8 +11,8 @@ from vllm.engine.arg_utils import EngineArgs
11 from vllm.platforms import current_platform
12 from vllm.usage.usage_lib import UsageContext
13 from vllm.v1.engine import EngineCoreRequest
14 -from vllm.v1.engine.async_llm import AsyncLLM
14 from vllm.v1.engine.core_client import EngineCoreClient
15 +from vllm.v1.executor.abstract import Executor
16
17 if not current_platform.is_cuda():
18 pytest.skip(reason="V1 currently only supported on CUDA.",
@@ -84,7 +84,7 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
84 engine_args = EngineArgs(model=MODEL_NAME, compilation_config=3)
85 vllm_config = engine_args.create_engine_config(
86 UsageContext.UNKNOWN_CONTEXT)
87 - executor_class = AsyncLLM._get_executor_cls(vllm_config)
87 + executor_class = Executor.get_class(vllm_config)
88 client = EngineCoreClient.make_client(
89 multiprocess_mode=multiprocessing_mode,
90 asyncio_mode=False,
@@ -152,7 +152,7 @@ async def test_engine_core_client_asyncio(monkeypatch):
152 engine_args = EngineArgs(model=MODEL_NAME)
153 vllm_config = engine_args.create_engine_config(
154 usage_context=UsageContext.UNKNOWN_CONTEXT)
155 - executor_class = AsyncLLM._get_executor_cls(vllm_config)
155 + executor_class = Executor.get_class(vllm_config)
156 client = EngineCoreClient.make_client(
157 multiprocess_mode=True,
158 asyncio_mode=True,
vllm/v1/engine/async_llm.py
+1 -20
@@ -22,7 +22,6 @@ from vllm.v1.engine.core_client import EngineCoreClient
22 from vllm.v1.engine.detokenizer import Detokenizer
23 from vllm.v1.engine.processor import Processor
24 from vllm.v1.executor.abstract import Executor
25 -from vllm.v1.executor.ray_utils import initialize_ray_cluster
25
26 logger = init_logger(__name__)
27
@@ -105,7 +104,7 @@ class AsyncLLM(EngineClient):
104 else:
105 vllm_config = engine_config
106
108 - executor_class = cls._get_executor_cls(vllm_config)
107 + executor_class = Executor.get_class(vllm_config)
108
109 # Create the AsyncLLM.
110 return cls(
@@ -127,24 +126,6 @@ class AsyncLLM(EngineClient):
126 if handler := getattr(self, "output_handler", None):
127 handler.cancel()
128
130 - @classmethod
131 - def _get_executor_cls(cls, vllm_config: VllmConfig) -> Type[Executor]:
132 - executor_class: Type[Executor]
133 - distributed_executor_backend = (
134 - vllm_config.parallel_config.distributed_executor_backend)
135 - if distributed_executor_backend == "ray":
136 - initialize_ray_cluster(vllm_config.parallel_config)
137 - from vllm.v1.executor.ray_executor import RayExecutor
138 - executor_class = RayExecutor
139 - elif distributed_executor_backend == "mp":
140 - from vllm.v1.executor.multiproc_executor import MultiprocExecutor
141 - executor_class = MultiprocExecutor
142 - else:
143 - assert (distributed_executor_backend is None)
144 - from vllm.v1.executor.uniproc_executor import UniprocExecutor
145 - executor_class = UniprocExecutor
146 - return executor_class
147 -
129 async def add_request(
130 self,
131 request_id: str,
vllm/v1/engine/llm_engine.py
+1 -19
@@ -89,7 +89,7 @@ class LLMEngine:
89
90 # Create the engine configs.
91 vllm_config = engine_args.create_engine_config(usage_context)
92 - executor_class = cls._get_executor_cls(vllm_config)
92 + executor_class = Executor.get_class(vllm_config)
93
94 if VLLM_ENABLE_V1_MULTIPROCESSING:
95 logger.debug("Enabling multiprocessing for LLMEngine.")
@@ -103,24 +103,6 @@ class LLMEngine:
103 stat_loggers=stat_loggers,
104 multiprocess_mode=enable_multiprocessing)
105
106 - @classmethod
107 - def _get_executor_cls(cls, vllm_config: VllmConfig) -> Type[Executor]:
108 - executor_class: Type[Executor]
109 - distributed_executor_backend = (
110 - vllm_config.parallel_config.distributed_executor_backend)
111 - if distributed_executor_backend == "ray":
112 - from vllm.v1.executor.ray_executor import RayExecutor
113 - executor_class = RayExecutor
114 - elif distributed_executor_backend == "mp":
115 - from vllm.v1.executor.multiproc_executor import MultiprocExecutor
116 - executor_class = MultiprocExecutor
117 - else:
118 - assert (distributed_executor_backend is None)
119 - from vllm.v1.executor.uniproc_executor import UniprocExecutor
120 - executor_class = UniprocExecutor
121 -
122 - return executor_class
123 -
106 def get_num_unfinished_requests(self) -> int:
107 return self.detokenizer.get_num_unfinished_requests()
108
vllm/v1/executor/abstract.py
+18 -1
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Tuple
2 +from typing import Tuple, Type
3
4 from vllm.config import VllmConfig
5 from vllm.v1.outputs import ModelRunnerOutput
@@ -8,6 +8,23 @@ from vllm.v1.outputs import ModelRunnerOutput
8 class Executor(ABC):
9 """Abstract class for executors."""
10
11 + @staticmethod
12 + def get_class(vllm_config: VllmConfig) -> Type["Executor"]:
13 + executor_class: Type[Executor]
14 + distributed_executor_backend = (
15 + vllm_config.parallel_config.distributed_executor_backend)
16 + if distributed_executor_backend == "ray":
17 + from vllm.v1.executor.ray_executor import RayExecutor
18 + executor_class = RayExecutor
19 + elif distributed_executor_backend == "mp":
20 + from vllm.v1.executor.multiproc_executor import MultiprocExecutor
21 + executor_class = MultiprocExecutor
22 + else:
23 + assert (distributed_executor_backend is None)
24 + from vllm.v1.executor.uniproc_executor import UniprocExecutor
25 + executor_class = UniprocExecutor
26 + return executor_class
27 +
28 @abstractmethod
29 def __init__(self, vllm_config: VllmConfig) -> None:
30 raise NotImplementedError