A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Frontend] Kill the server on engine death (#6594)
Signed-off-by: Joe Runde <joe@joerun.de> Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Aug 8, 2024 at 10:47 UTC
21b9c49aa37c7ba08590a99b0d4f15f86439c8f9
8 files changed
+136
-14
tests/entrypoints/openai/test_shutdown.py
new
+47
@@ -0,0 +1,47 @@
1
+import json
2
+import os
3
+
4
+import openai
5
+import pytest
6
+
7
+from ...utils import RemoteOpenAIServer
8
+
9
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
10
+
11
+
12
+@pytest.mark.asyncio
13
+async def test_shutdown_on_engine_failure(tmp_path):
14
+ # Use a bad adapter to crash the engine
15
+ # (This test will fail when that bug is fixed)
16
+ adapter_path = tmp_path / "bad_adapter"
17
+ os.mkdir(adapter_path)
18
+ with open(adapter_path / "adapter_model_config.json", "w") as f:
19
+ json.dump({"not": "real"}, f)
20
+ with open(adapter_path / "adapter_model.safetensors", "wb") as f:
21
+ f.write(b"this is fake")
22
+
23
+ # dtype, max-len etc set so that this can run in CI
24
+ args = [
25
+ "--dtype",
26
+ "bfloat16",
27
+ "--max-model-len",
28
+ "8192",
29
+ "--enforce-eager",
30
+ "--max-num-seqs",
31
+ "128",
32
+ "--enable-lora",
33
+ "--lora-modules",
34
+ f"bad-adapter={tmp_path / 'bad_adapter'}",
35
+ ]
36
+
37
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
38
+ client = remote_server.get_async_client()
39
+
40
+ with pytest.raises(openai.APIConnectionError):
41
+ # This crashes the engine
42
+ await client.completions.create(model="bad-adapter",
43
+ prompt="Hello, my name is")
44
+
45
+ # Now the server should shut down
46
+ return_code = remote_server.proc.wait(timeout=1)
47
+ assert return_code is not None
vllm/engine/async_llm_engine.py
+4
-2
@@ -58,7 +58,7 @@ def _log_task_completion(task: asyncio.Task,
58
error_callback(exception)
59
raise AsyncEngineDeadError(
60
"Task finished unexpectedly. This should never happen! "
61
- "Please open an issue on Github. See stack trace above for the"
61
+ "Please open an issue on Github. See stack trace above for the "
62
"actual cause.") from e
63
64
@@ -132,7 +132,9 @@ class RequestTracker:
132
self._request_streams[request_id].put(exc)
133
self.abort_request(request_id)
134
else:
135
- for rid, stream in self._request_streams.items():
135
+ # NB: list() used here because self.abort_request pops the stream
136
+ # out of self._request_streams, so we can't iterate on it directly
137
+ for rid, stream in list(self._request_streams.items()):
138
stream.put(exc)
139
self.abort_request(rid)
140
vllm/entrypoints/api_server.py
+1
@@ -118,6 +118,7 @@ async def run_server(args: Namespace,
118
119
shutdown_task = await serve_http(
120
app,
121
+ engine=engine,
122
host=args.host,
123
port=args.port,
124
log_level=args.log_level,
vllm/entrypoints/launcher.py
+42
-2
@@ -1,16 +1,21 @@
1
import asyncio
2
import signal
3
+from http import HTTPStatus
4
from typing import Any
5
6
import uvicorn
6
-from fastapi import FastAPI
7
+from fastapi import FastAPI, Response
8
9
+from vllm import envs
10
+from vllm.engine.async_llm_engine import AsyncEngineDeadError
11
+from vllm.engine.protocol import AsyncEngineClient
12
from vllm.logger import init_logger
13
14
logger = init_logger(__name__)
15
16
13
-async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
17
+async def serve_http(app: FastAPI, engine: AsyncEngineClient,
18
+ **uvicorn_kwargs: Any):
19
logger.info("Available routes are:")
20
for route in app.routes:
21
methods = getattr(route, "methods", None)
@@ -23,6 +28,7 @@ async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
28
29
config = uvicorn.Config(app, **uvicorn_kwargs)
30
server = uvicorn.Server(config)
31
+ _add_shutdown_handlers(app, server, engine)
32
33
loop = asyncio.get_running_loop()
34
@@ -44,3 +50,37 @@ async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
50
except asyncio.CancelledError:
51
logger.info("Gracefully stopping http server")
52
return server.shutdown()
53
+
54
+
55
+def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server,
56
+ engine: AsyncEngineClient) -> None:
57
+ """Adds handlers for fatal errors that should crash the server"""
58
+
59
+ @app.exception_handler(RuntimeError)
60
+ async def runtime_error_handler(_, __):
61
+ """On generic runtime error, check to see if the engine has died.
62
+ It probably has, in which case the server will no longer be able to
63
+ handle requests. Trigger a graceful shutdown with a SIGTERM."""
64
+ if (not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine.errored
65
+ and not engine.is_running):
66
+ logger.fatal("AsyncLLMEngine has failed, terminating server "
67
+ "process")
68
+ # See discussions here on shutting down a uvicorn server
69
+ # https://github.com/encode/uvicorn/discussions/1103
70
+ # In this case we cannot await the server shutdown here because
71
+ # this handler must first return to close the connection for
72
+ # this request.
73
+ server.should_exit = True
74
+
75
+ return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
76
+
77
+ @app.exception_handler(AsyncEngineDeadError)
78
+ async def engine_dead_handler(_, __):
79
+ """Kill the server if the async engine is already dead. It will
80
+ not handle any further requests."""
81
+ if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH:
82
+ logger.fatal("AsyncLLMEngine is already dead, terminating server "
83
+ "process")
84
+ server.should_exit = True
85
+
86
+ return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
vllm/entrypoints/openai/api_server.py
+1
@@ -357,6 +357,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:
357
358
shutdown_task = await serve_http(
359
app,
360
+ engine=async_engine_client,
361
host=args.host,
362
port=args.port,
363
log_level=args.uvicorn_log_level,
vllm/entrypoints/openai/rpc/client.py
+24
-2
@@ -33,6 +33,7 @@ class AsyncEngineRPCClient:
33
34
# Wait until server is ready.
35
await self.wait_for_server()
36
+ self._errored = False
37
38
# Get the configs.
39
self.model_config = await self._get_model_config_rpc()
@@ -169,7 +170,7 @@ class AsyncEngineRPCClient:
170
expected_type=SchedulerConfig,
171
error_message="Could not get SchedulerConfig from RPC Server")
172
172
- async def _get_lora_config_rpc(self):
173
+ async def _get_lora_config_rpc(self) -> LoRAConfig:
174
"""Get LoRAConfig from the RPCServer"""
175
176
return await self._send_get_data_rpc_request(
@@ -177,7 +178,7 @@ class AsyncEngineRPCClient:
178
expected_type=LoRAConfig,
179
error_message="Could not get LoRAConfig from RPC Server")
180
180
- async def _is_tracing_enabled_rpc(self) -> ParallelConfig:
181
+ async def _is_tracing_enabled_rpc(self) -> bool:
182
"""Get is_tracing_enabled flag from the RPCServer"""
183
184
return await self._send_get_data_rpc_request(
@@ -200,6 +201,18 @@ class AsyncEngineRPCClient:
201
request=RPCUtilityRequest.DO_LOG_STATS,
202
error_message="RPCRequest DO_LOG_STATS failed.")
203
204
+ @property
205
+ def is_running(self) -> bool:
206
+ return not self._errored
207
+
208
+ @property
209
+ def is_stopped(self) -> bool:
210
+ return self._errored
211
+
212
+ @property
213
+ def errored(self) -> bool:
214
+ return self._errored
215
+
216
async def generate(
217
self,
218
inputs: PromptInputs,
@@ -233,6 +246,15 @@ class AsyncEngineRPCClient:
246
request_output = cloudpickle.loads(message)
247
248
if isinstance(request_output, Exception):
249
+ # On exception, check if the server is still healthy.
250
+ # Use this to set the sync `is_running` and `errored`
251
+ # properties.
252
+ try:
253
+ await self.check_health()
254
+ except Exception:
255
+ self._errored = True
256
+ # NB: do before raising here so that the flag is set
257
+ # by the time the caller receives this exception
258
raise request_output
259
260
finished = request_output.finished
vllm/entrypoints/openai/rpc/server.py
+11
-8
@@ -96,14 +96,17 @@ class AsyncEngineRPCServer:
96
97
async def abort(self, identity, request: RPCAbortRequest):
98
"""Abort request and notify the client of success."""
99
- # Abort the request in the llm engine.
100
- await self.engine.abort(request.request_id)
101
-
102
- # Send confirmation to the client.
103
- await self.socket.send_multipart([
104
- identity,
105
- cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
106
- ])
99
+ try:
100
+ # Abort the request in the llm engine.
101
+ await self.engine.abort(request.request_id)
102
+ except Exception:
103
+ logger.warning("Failed to abort request %s", request.request_id)
104
+ finally:
105
+ # Send confirmation to the client.
106
+ await self.socket.send_multipart([
107
+ identity,
108
+ cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
109
+ ])
110
111
async def generate(self, identity, generate_request: RPCGenerateRequest):
112
try:
vllm/envs.py
+6
@@ -49,6 +49,7 @@ if TYPE_CHECKING:
49
NVCC_THREADS: Optional[str] = None
50
VLLM_USE_PRECOMPILED: bool = False
51
VLLM_NO_DEPRECATION_WARNING: bool = False
52
+ VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
53
CMAKE_BUILD_TYPE: Optional[str] = None
54
VERBOSE: bool = False
55
VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
@@ -335,6 +336,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
336
"VLLM_NO_DEPRECATION_WARNING":
337
lambda: bool(int(os.getenv("VLLM_NO_DEPRECATION_WARNING", "0"))),
338
339
+ # If set, the OpenAI API server will stay alive even after the underlying
340
+ # AsyncLLMEngine errors and stops serving requests
341
+ "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH":
342
+ lambda: bool(os.getenv("VLLM_KEEP_ALIVE_ON_ENGINE_DEATH", 0)),
343
+
344
# If the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN is set, it allows
345
# the user to specify a max sequence length greater than
346
# the max length derived from the model's config.json.