@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 21b9c49aa3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Frontend] Kill the server on engine death (#6594)

Signed-off-by: Joe Runde <joe@joerun.de> Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Aug 8, 2024 at 10:47 UTC 21b9c49aa37c7ba08590a99b0d4f15f86439c8f9
8 files changed +136 -14
tests/entrypoints/openai/test_shutdown.py new
+47
@@ -0,0 +1,47 @@
1 +import json
2 +import os
3 +
4 +import openai
5 +import pytest
6 +
7 +from ...utils import RemoteOpenAIServer
8 +
9 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
10 +
11 +
12 +@pytest.mark.asyncio
13 +async def test_shutdown_on_engine_failure(tmp_path):
14 + # Use a bad adapter to crash the engine
15 + # (This test will fail when that bug is fixed)
16 + adapter_path = tmp_path / "bad_adapter"
17 + os.mkdir(adapter_path)
18 + with open(adapter_path / "adapter_model_config.json", "w") as f:
19 + json.dump({"not": "real"}, f)
20 + with open(adapter_path / "adapter_model.safetensors", "wb") as f:
21 + f.write(b"this is fake")
22 +
23 + # dtype, max-len etc set so that this can run in CI
24 + args = [
25 + "--dtype",
26 + "bfloat16",
27 + "--max-model-len",
28 + "8192",
29 + "--enforce-eager",
30 + "--max-num-seqs",
31 + "128",
32 + "--enable-lora",
33 + "--lora-modules",
34 + f"bad-adapter={tmp_path / 'bad_adapter'}",
35 + ]
36 +
37 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
38 + client = remote_server.get_async_client()
39 +
40 + with pytest.raises(openai.APIConnectionError):
41 + # This crashes the engine
42 + await client.completions.create(model="bad-adapter",
43 + prompt="Hello, my name is")
44 +
45 + # Now the server should shut down
46 + return_code = remote_server.proc.wait(timeout=1)
47 + assert return_code is not None
vllm/engine/async_llm_engine.py
+4 -2
@@ -58,7 +58,7 @@ def _log_task_completion(task: asyncio.Task,
58 error_callback(exception)
59 raise AsyncEngineDeadError(
60 "Task finished unexpectedly. This should never happen! "
61 - "Please open an issue on Github. See stack trace above for the"
61 + "Please open an issue on Github. See stack trace above for the "
62 "actual cause.") from e
63
64
@@ -132,7 +132,9 @@ class RequestTracker:
132 self._request_streams[request_id].put(exc)
133 self.abort_request(request_id)
134 else:
135 - for rid, stream in self._request_streams.items():
135 + # NB: list() used here because self.abort_request pops the stream
136 + # out of self._request_streams, so we can't iterate on it directly
137 + for rid, stream in list(self._request_streams.items()):
138 stream.put(exc)
139 self.abort_request(rid)
140
vllm/entrypoints/api_server.py
+1
@@ -118,6 +118,7 @@ async def run_server(args: Namespace,
118
119 shutdown_task = await serve_http(
120 app,
121 + engine=engine,
122 host=args.host,
123 port=args.port,
124 log_level=args.log_level,
vllm/entrypoints/launcher.py
+42 -2
@@ -1,16 +1,21 @@
1 import asyncio
2 import signal
3 +from http import HTTPStatus
4 from typing import Any
5
6 import uvicorn
6 -from fastapi import FastAPI
7 +from fastapi import FastAPI, Response
8
9 +from vllm import envs
10 +from vllm.engine.async_llm_engine import AsyncEngineDeadError
11 +from vllm.engine.protocol import AsyncEngineClient
12 from vllm.logger import init_logger
13
14 logger = init_logger(__name__)
15
16
13 -async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
17 +async def serve_http(app: FastAPI, engine: AsyncEngineClient,
18 + **uvicorn_kwargs: Any):
19 logger.info("Available routes are:")
20 for route in app.routes:
21 methods = getattr(route, "methods", None)
@@ -23,6 +28,7 @@ async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
28
29 config = uvicorn.Config(app, **uvicorn_kwargs)
30 server = uvicorn.Server(config)
31 + _add_shutdown_handlers(app, server, engine)
32
33 loop = asyncio.get_running_loop()
34
@@ -44,3 +50,37 @@ async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
50 except asyncio.CancelledError:
51 logger.info("Gracefully stopping http server")
52 return server.shutdown()
53 +
54 +
55 +def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server,
56 + engine: AsyncEngineClient) -> None:
57 + """Adds handlers for fatal errors that should crash the server"""
58 +
59 + @app.exception_handler(RuntimeError)
60 + async def runtime_error_handler(_, __):
61 + """On generic runtime error, check to see if the engine has died.
62 + It probably has, in which case the server will no longer be able to
63 + handle requests. Trigger a graceful shutdown with a SIGTERM."""
64 + if (not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine.errored
65 + and not engine.is_running):
66 + logger.fatal("AsyncLLMEngine has failed, terminating server "
67 + "process")
68 + # See discussions here on shutting down a uvicorn server
69 + # https://github.com/encode/uvicorn/discussions/1103
70 + # In this case we cannot await the server shutdown here because
71 + # this handler must first return to close the connection for
72 + # this request.
73 + server.should_exit = True
74 +
75 + return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
76 +
77 + @app.exception_handler(AsyncEngineDeadError)
78 + async def engine_dead_handler(_, __):
79 + """Kill the server if the async engine is already dead. It will
80 + not handle any further requests."""
81 + if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH:
82 + logger.fatal("AsyncLLMEngine is already dead, terminating server "
83 + "process")
84 + server.should_exit = True
85 +
86 + return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
vllm/entrypoints/openai/api_server.py
+1
@@ -357,6 +357,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:
357
358 shutdown_task = await serve_http(
359 app,
360 + engine=async_engine_client,
361 host=args.host,
362 port=args.port,
363 log_level=args.uvicorn_log_level,
vllm/entrypoints/openai/rpc/client.py
+24 -2
@@ -33,6 +33,7 @@ class AsyncEngineRPCClient:
33
34 # Wait until server is ready.
35 await self.wait_for_server()
36 + self._errored = False
37
38 # Get the configs.
39 self.model_config = await self._get_model_config_rpc()
@@ -169,7 +170,7 @@ class AsyncEngineRPCClient:
170 expected_type=SchedulerConfig,
171 error_message="Could not get SchedulerConfig from RPC Server")
172
172 - async def _get_lora_config_rpc(self):
173 + async def _get_lora_config_rpc(self) -> LoRAConfig:
174 """Get LoRAConfig from the RPCServer"""
175
176 return await self._send_get_data_rpc_request(
@@ -177,7 +178,7 @@ class AsyncEngineRPCClient:
178 expected_type=LoRAConfig,
179 error_message="Could not get LoRAConfig from RPC Server")
180
180 - async def _is_tracing_enabled_rpc(self) -> ParallelConfig:
181 + async def _is_tracing_enabled_rpc(self) -> bool:
182 """Get is_tracing_enabled flag from the RPCServer"""
183
184 return await self._send_get_data_rpc_request(
@@ -200,6 +201,18 @@ class AsyncEngineRPCClient:
201 request=RPCUtilityRequest.DO_LOG_STATS,
202 error_message="RPCRequest DO_LOG_STATS failed.")
203
204 + @property
205 + def is_running(self) -> bool:
206 + return not self._errored
207 +
208 + @property
209 + def is_stopped(self) -> bool:
210 + return self._errored
211 +
212 + @property
213 + def errored(self) -> bool:
214 + return self._errored
215 +
216 async def generate(
217 self,
218 inputs: PromptInputs,
@@ -233,6 +246,15 @@ class AsyncEngineRPCClient:
246 request_output = cloudpickle.loads(message)
247
248 if isinstance(request_output, Exception):
249 + # On exception, check if the server is still healthy.
250 + # Use this to set the sync `is_running` and `errored`
251 + # properties.
252 + try:
253 + await self.check_health()
254 + except Exception:
255 + self._errored = True
256 + # NB: do before raising here so that the flag is set
257 + # by the time the caller receives this exception
258 raise request_output
259
260 finished = request_output.finished
vllm/entrypoints/openai/rpc/server.py
+11 -8
@@ -96,14 +96,17 @@ class AsyncEngineRPCServer:
96
97 async def abort(self, identity, request: RPCAbortRequest):
98 """Abort request and notify the client of success."""
99 - # Abort the request in the llm engine.
100 - await self.engine.abort(request.request_id)
101 -
102 - # Send confirmation to the client.
103 - await self.socket.send_multipart([
104 - identity,
105 - cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
106 - ])
99 + try:
100 + # Abort the request in the llm engine.
101 + await self.engine.abort(request.request_id)
102 + except Exception:
103 + logger.warning("Failed to abort request %s", request.request_id)
104 + finally:
105 + # Send confirmation to the client.
106 + await self.socket.send_multipart([
107 + identity,
108 + cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
109 + ])
110
111 async def generate(self, identity, generate_request: RPCGenerateRequest):
112 try:
vllm/envs.py
+6
@@ -49,6 +49,7 @@ if TYPE_CHECKING:
49 NVCC_THREADS: Optional[str] = None
50 VLLM_USE_PRECOMPILED: bool = False
51 VLLM_NO_DEPRECATION_WARNING: bool = False
52 + VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
53 CMAKE_BUILD_TYPE: Optional[str] = None
54 VERBOSE: bool = False
55 VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
@@ -335,6 +336,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
336 "VLLM_NO_DEPRECATION_WARNING":
337 lambda: bool(int(os.getenv("VLLM_NO_DEPRECATION_WARNING", "0"))),
338
339 + # If set, the OpenAI API server will stay alive even after the underlying
340 + # AsyncLLMEngine errors and stops serving requests
341 + "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH":
342 + lambda: bool(os.getenv("VLLM_KEEP_ALIVE_ON_ENGINE_DEATH", 0)),
343 +
344 # If the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN is set, it allows
345 # the user to specify a max sequence length greater than
346 # the max length derived from the model's config.json.