@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 584f0ae40d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Make AsyncLLMEngine v1-v0 opaque (#11383)

Signed-off-by: Ricky Xu <xuchen727@hotmail.com>

Ricky Xu committed Dec 20, 2024 at 23:14 UTC 584f0ae40d6f64a7097525f04feb236e94ad37fd
3 files changed +9 -10
vllm/engine/async_llm_engine.py
+7
@@ -1256,3 +1256,10 @@ class AsyncLLMEngine(EngineClient):
1256 self.engine.model_executor.stop_profile()
1257 else:
1258 self.engine.model_executor._run_workers("stop_profile")
1259 +
1260 +
1261 +# TODO(v1): Remove this class proxy when V1 goes default.
1262 +if envs.VLLM_USE_V1:
1263 + from vllm.v1.engine.async_llm import AsyncLLM
1264 +
1265 + AsyncLLMEngine = AsyncLLM # type: ignore
vllm/entrypoints/openai/api_server.py
+1 -5
@@ -27,6 +27,7 @@ from typing_extensions import assert_never
27 import vllm.envs as envs
28 from vllm.config import ModelConfig
29 from vllm.engine.arg_utils import AsyncEngineArgs
30 +from vllm.engine.async_llm_engine import AsyncLLMEngine # type: ignore
31 from vllm.engine.multiprocessing.client import MQLLMEngineClient
32 from vllm.engine.multiprocessing.engine import run_mp_engine
33 from vllm.engine.protocol import EngineClient
@@ -66,11 +67,6 @@ from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
67 is_valid_ipv6_address)
68 from vllm.version import __version__ as VLLM_VERSION
69
69 -if envs.VLLM_USE_V1:
70 - from vllm.v1.engine.async_llm import AsyncLLMEngine # type: ignore
71 -else:
72 - from vllm.engine.async_llm_engine import AsyncLLMEngine # type: ignore
73 -
70 TIMEOUT_KEEP_ALIVE = 5 # seconds
71
72 prometheus_multiproc_dir: tempfile.TemporaryDirectory
vllm/v1/engine/async_llm.py
+1 -5
@@ -98,7 +98,7 @@ class AsyncLLM(EngineClient):
98 start_engine_loop: bool = True,
99 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
100 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
101 - ) -> "AsyncLLMEngine":
101 + ) -> "AsyncLLM":
102 """Create an AsyncLLM from the EngineArgs."""
103
104 # Create the engine configs.
@@ -386,7 +386,3 @@ class AsyncLLM(EngineClient):
386 @property
387 def dead_error(self) -> BaseException:
388 return Exception() # TODO: implement
389 -
390 -
391 -# Retain V0 name for backwards compatibility.
392 -AsyncLLMEngine = AsyncLLM