A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] [6/N] API Server: Better Shutdown (#11586)
Robert Shaw committed
Dec 30, 2024 at 10:51 UTC
5886aa496e8fa31c9180bcfc8e89faaa8899907d
3 files changed
+40
-45
vllm/entrypoints/openai/api_server.py
+12
-32
@@ -68,7 +68,7 @@ from vllm.entrypoints.utils import with_cancellation
68
from vllm.logger import init_logger
69
from vllm.usage.usage_lib import UsageContext
70
from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
71
- is_valid_ipv6_address, kill_process_tree, set_ulimit)
71
+ is_valid_ipv6_address, set_ulimit)
72
from vllm.version import __version__ as VLLM_VERSION
73
74
TIMEOUT_KEEP_ALIVE = 5 # seconds
@@ -133,32 +133,21 @@ async def build_async_engine_client_from_engine_args(
133
Returns the Client or None if the creation failed.
134
"""
135
136
- # Fall back
137
- # TODO: fill out feature matrix.
136
+ # AsyncLLMEngine.
137
if (MQLLMEngineClient.is_unsupported_config(engine_args)
138
or envs.VLLM_USE_V1 or disable_frontend_multiprocessing):
140
- engine_config = engine_args.create_engine_config(
141
- UsageContext.OPENAI_API_SERVER)
142
- uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config),
143
- "uses_ray", False)
144
-
145
- build_engine = partial(AsyncLLMEngine.from_engine_args,
146
- engine_args=engine_args,
147
- engine_config=engine_config,
148
- usage_context=UsageContext.OPENAI_API_SERVER)
149
- if uses_ray:
150
- # Must run in main thread with ray for its signal handlers to work
151
- engine_client = build_engine()
152
- else:
153
- engine_client = await asyncio.get_running_loop().run_in_executor(
154
- None, build_engine)
139
156
- yield engine_client
157
- if hasattr(engine_client, "shutdown"):
158
- engine_client.shutdown()
159
- return
140
+ engine_client: Optional[EngineClient] = None
141
+ try:
142
+ engine_client = AsyncLLMEngine.from_engine_args(
143
+ engine_args=engine_args,
144
+ usage_context=UsageContext.OPENAI_API_SERVER)
145
+ yield engine_client
146
+ finally:
147
+ if engine_client and hasattr(engine_client, "shutdown"):
148
+ engine_client.shutdown()
149
161
- # Otherwise, use the multiprocessing AsyncLLMEngine.
150
+ # MQLLMEngine.
151
else:
152
if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
153
# Make TemporaryDirectory for prometheus multiprocessing
@@ -737,15 +726,6 @@ async def run_server(args, **uvicorn_kwargs) -> None:
726
727
signal.signal(signal.SIGTERM, signal_handler)
728
740
- # The child processes will send SIGQUIT to this process when
741
- # any error happens. This process then clean up the whole tree.
742
- # TODO(rob): move this into AsyncLLM.__init__ once we remove
743
- # the context manager below.
744
- def sigquit_handler(signum, frame):
745
- kill_process_tree(os.getpid())
746
-
747
- signal.signal(signal.SIGQUIT, sigquit_handler)
748
-
729
async with build_async_engine_client(args) as engine_client:
730
app = build_app(args)
731
vllm/v1/engine/async_llm.py
+22
-3
@@ -1,4 +1,6 @@
1
import asyncio
2
+import os
3
+import signal
4
from typing import AsyncGenerator, Dict, List, Mapping, Optional, Type, Union
5
6
from vllm.config import ModelConfig, VllmConfig
@@ -16,6 +18,7 @@ from vllm.sampling_params import SamplingParams
18
from vllm.transformers_utils.tokenizer import AnyTokenizer
19
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
20
from vllm.usage.usage_lib import UsageContext
21
+from vllm.utils import kill_process_tree
22
from vllm.v1.engine.core_client import EngineCoreClient
23
from vllm.v1.engine.detokenizer import Detokenizer
24
from vllm.v1.engine.processor import Processor
@@ -38,6 +41,22 @@ class AsyncLLM(EngineClient):
41
log_requests: bool = True,
42
start_engine_loop: bool = True,
43
) -> None:
44
+
45
+ # The child processes will send SIGQUIT when unrecoverable
46
+ # errors happen. We kill the process tree here so that the
47
+ # stack trace is very evident.
48
+ # TODO: rather than killing the main process, we should
49
+ # figure out how to raise an AsyncEngineDeadError and
50
+ # handle at the API server level so we can return a better
51
+ # error code to the clients calling VLLM.
52
+ def sigquit_handler(signum, frame):
53
+ logger.fatal(
54
+ "AsyncLLM got SIGQUIT from worker processes, shutting "
55
+ "down. See stack trace above for root cause issue.")
56
+ kill_process_tree(os.getpid())
57
+
58
+ signal.signal(signal.SIGQUIT, sigquit_handler)
59
+
60
assert start_engine_loop
61
62
self.log_requests = log_requests
@@ -276,9 +295,9 @@ class AsyncLLM(EngineClient):
295
# 4) Abort any requests that finished due to stop strings.
296
await self.engine_core.abort_requests_async(reqs_to_abort)
297
279
- except BaseException as e:
280
- logger.error(e)
281
- raise e
298
+ except Exception as e:
299
+ logger.exception("EngineCore output handler hit an error: %s", e)
300
+ kill_process_tree(os.getpid())
301
302
async def abort(self, request_id: str) -> None:
303
"""Abort RequestId in self, detokenizer, and engine core."""
vllm/v1/engine/core_client.py
+6
-10
@@ -6,7 +6,7 @@ import zmq.asyncio
6
7
from vllm.config import VllmConfig
8
from vllm.logger import init_logger
9
-from vllm.utils import get_open_zmq_ipc_path
9
+from vllm.utils import get_open_zmq_ipc_path, make_zmq_socket
10
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
11
EngineCoreProfile, EngineCoreRequest,
12
EngineCoreRequestType, EngineCoreRequestUnion)
@@ -144,17 +144,13 @@ class MPClient(EngineCoreClient):
144
else:
145
self.ctx = zmq.Context() # type: ignore[attr-defined]
146
147
- # Path for IPC.
147
+ # Paths and sockets for IPC.
148
output_path = get_open_zmq_ipc_path()
149
input_path = get_open_zmq_ipc_path()
150
-
151
- # Get output (EngineCoreOutput) from EngineCore.
152
- self.output_socket = self.ctx.socket(zmq.constants.PULL)
153
- self.output_socket.connect(output_path)
154
-
155
- # Send input (EngineCoreRequest) to EngineCore.
156
- self.input_socket = self.ctx.socket(zmq.constants.PUSH)
157
- self.input_socket.bind(input_path)
150
+ self.output_socket = make_zmq_socket(self.ctx, output_path,
151
+ zmq.constants.PULL)
152
+ self.input_socket = make_zmq_socket(self.ctx, input_path,
153
+ zmq.constants.PUSH)
154
155
# Start EngineCore in background process.
156
self.proc_handle: Optional[BackgroundProcHandle]