A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Make AsyncLLMEngine v1-v0 opaque (#11383)
Signed-off-by: Ricky Xu <xuchen727@hotmail.com>
Ricky Xu committed
Dec 20, 2024 at 23:14 UTC
584f0ae40d6f64a7097525f04feb236e94ad37fd
3 files changed
+9
-10
vllm/engine/async_llm_engine.py
+7
@@ -1256,3 +1256,10 @@ class AsyncLLMEngine(EngineClient):
1256
self.engine.model_executor.stop_profile()
1257
else:
1258
self.engine.model_executor._run_workers("stop_profile")
1259
+
1260
+
1261
+# TODO(v1): Remove this class proxy when V1 goes default.
1262
+if envs.VLLM_USE_V1:
1263
+ from vllm.v1.engine.async_llm import AsyncLLM
1264
+
1265
+ AsyncLLMEngine = AsyncLLM # type: ignore
vllm/entrypoints/openai/api_server.py
+1
-5
@@ -27,6 +27,7 @@ from typing_extensions import assert_never
27
import vllm.envs as envs
28
from vllm.config import ModelConfig
29
from vllm.engine.arg_utils import AsyncEngineArgs
30
+from vllm.engine.async_llm_engine import AsyncLLMEngine # type: ignore
31
from vllm.engine.multiprocessing.client import MQLLMEngineClient
32
from vllm.engine.multiprocessing.engine import run_mp_engine
33
from vllm.engine.protocol import EngineClient
@@ -66,11 +67,6 @@ from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
67
is_valid_ipv6_address)
68
from vllm.version import __version__ as VLLM_VERSION
69
69
-if envs.VLLM_USE_V1:
70
- from vllm.v1.engine.async_llm import AsyncLLMEngine # type: ignore
71
-else:
72
- from vllm.engine.async_llm_engine import AsyncLLMEngine # type: ignore
73
-
70
TIMEOUT_KEEP_ALIVE = 5 # seconds
71
72
prometheus_multiproc_dir: tempfile.TemporaryDirectory
vllm/v1/engine/async_llm.py
+1
-5
@@ -98,7 +98,7 @@ class AsyncLLM(EngineClient):
98
start_engine_loop: bool = True,
99
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
100
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
101
- ) -> "AsyncLLMEngine":
101
+ ) -> "AsyncLLM":
102
"""Create an AsyncLLM from the EngineArgs."""
103
104
# Create the engine configs.
@@ -386,7 +386,3 @@ class AsyncLLM(EngineClient):
386
@property
387
def dead_error(self) -> BaseException:
388
return Exception() # TODO: implement
389
-
390
-
391
-# Retain V0 name for backwards compatibility.
392
-AsyncLLMEngine = AsyncLLM