A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Frontend] Factor out code for running uvicorn (#6828)
Cyrus Leung committed
Jul 27, 2024 at 09:58 UTC
981b0d567355063d5453e382a85970cae083c615
4 files changed
+116
-75
vllm/entrypoints/api_server.py
+50
-24
@@ -5,12 +5,12 @@ For production use, we recommend using our OpenAI compatible server.
5
We are also not going to accept PRs modifying this file, please
6
change `vllm/entrypoints/openai/api_server.py` instead.
7
"""
8
-
8
+import asyncio
9
import json
10
import ssl
11
-from typing import AsyncGenerator
11
+from argparse import Namespace
12
+from typing import Any, AsyncGenerator, Optional
13
13
-import uvicorn
14
from fastapi import FastAPI, Request
15
from fastapi.responses import JSONResponse, Response, StreamingResponse
16
@@ -18,8 +18,10 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.async_llm_engine import AsyncLLMEngine
19
from vllm.logger import init_logger
20
from vllm.sampling_params import SamplingParams
21
+from vllm.server import serve_http
22
from vllm.usage.usage_lib import UsageContext
23
from vllm.utils import FlexibleArgumentParser, random_uuid
24
+from vllm.version import __version__ as VLLM_VERSION
25
26
logger = init_logger("vllm.entrypoints.api_server")
27
@@ -81,6 +83,50 @@ async def generate(request: Request) -> Response:
83
return JSONResponse(ret)
84
85
86
+def build_app(args: Namespace) -> FastAPI:
87
+ global app
88
+
89
+ app.root_path = args.root_path
90
+ return app
91
+
92
+
93
+async def init_app(
94
+ args: Namespace,
95
+ llm_engine: Optional[AsyncLLMEngine] = None,
96
+) -> FastAPI:
97
+ app = build_app(args)
98
+
99
+ global engine
100
+
101
+ engine_args = AsyncEngineArgs.from_cli_args(args)
102
+ engine = (llm_engine
103
+ if llm_engine is not None else AsyncLLMEngine.from_engine_args(
104
+ engine_args, usage_context=UsageContext.API_SERVER))
105
+
106
+ return app
107
+
108
+
109
+async def run_server(args: Namespace,
110
+ llm_engine: Optional[AsyncLLMEngine] = None,
111
+ **uvicorn_kwargs: Any) -> None:
112
+ logger.info("vLLM API server version %s", VLLM_VERSION)
113
+ logger.info("args: %s", args)
114
+
115
+ app = await init_app(args, llm_engine)
116
+ await serve_http(
117
+ app,
118
+ host=args.host,
119
+ port=args.port,
120
+ log_level=args.log_level,
121
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
122
+ ssl_keyfile=args.ssl_keyfile,
123
+ ssl_certfile=args.ssl_certfile,
124
+ ssl_ca_certs=args.ssl_ca_certs,
125
+ ssl_cert_reqs=args.ssl_cert_reqs,
126
+ **uvicorn_kwargs,
127
+ )
128
+
129
+
130
if __name__ == "__main__":
131
parser = FlexibleArgumentParser()
132
parser.add_argument("--host", type=str, default=None)
@@ -105,25 +151,5 @@ if __name__ == "__main__":
151
parser.add_argument("--log-level", type=str, default="debug")
152
parser = AsyncEngineArgs.add_cli_args(parser)
153
args = parser.parse_args()
108
- engine_args = AsyncEngineArgs.from_cli_args(args)
109
- engine = AsyncLLMEngine.from_engine_args(
110
- engine_args, usage_context=UsageContext.API_SERVER)
111
-
112
- app.root_path = args.root_path
154
114
- logger.info("Available routes are:")
115
- for route in app.routes:
116
- if not hasattr(route, 'methods'):
117
- continue
118
- methods = ', '.join(route.methods)
119
- logger.info("Route: %s, Methods: %s", route.path, methods)
120
-
121
- uvicorn.run(app,
122
- host=args.host,
123
- port=args.port,
124
- log_level=args.log_level,
125
- timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
126
- ssl_keyfile=args.ssl_keyfile,
127
- ssl_certfile=args.ssl_certfile,
128
- ssl_ca_certs=args.ssl_ca_certs,
129
- ssl_cert_reqs=args.ssl_cert_reqs)
155
+ asyncio.run(run_server(args))
vllm/entrypoints/openai/api_server.py
+21
-51
@@ -2,14 +2,12 @@ import asyncio
2
import importlib
3
import inspect
4
import re
5
-import signal
5
+from argparse import Namespace
6
from contextlib import asynccontextmanager
7
from http import HTTPStatus
8
-from typing import Optional, Set
8
+from typing import Any, Optional, Set
9
10
-import fastapi
11
-import uvicorn
12
-from fastapi import APIRouter, Request
10
+from fastapi import APIRouter, FastAPI, Request
11
from fastapi.exceptions import RequestValidationError
12
from fastapi.middleware.cors import CORSMiddleware
13
from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -38,6 +36,7 @@ from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
36
from vllm.entrypoints.openai.serving_tokenization import (
37
OpenAIServingTokenization)
38
from vllm.logger import init_logger
39
+from vllm.server import serve_http
40
from vllm.usage.usage_lib import UsageContext
41
from vllm.utils import FlexibleArgumentParser
42
from vllm.version import __version__ as VLLM_VERSION
@@ -57,7 +56,7 @@ _running_tasks: Set[asyncio.Task] = set()
56
57
58
@asynccontextmanager
60
-async def lifespan(app: fastapi.FastAPI):
59
+async def lifespan(app: FastAPI):
60
61
async def _force_log():
62
while True:
@@ -75,7 +74,7 @@ async def lifespan(app: fastapi.FastAPI):
74
router = APIRouter()
75
76
78
-def mount_metrics(app: fastapi.FastAPI):
77
+def mount_metrics(app: FastAPI):
78
# Add prometheus asgi middleware to route /metrics requests
79
metrics_route = Mount("/metrics", make_asgi_app())
80
# Workaround for 307 Redirect for /metrics
@@ -165,8 +164,8 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
164
return JSONResponse(content=generator.model_dump())
165
166
168
-def build_app(args):
169
- app = fastapi.FastAPI(lifespan=lifespan)
167
+def build_app(args: Namespace) -> FastAPI:
168
+ app = FastAPI(lifespan=lifespan)
169
app.include_router(router)
170
app.root_path = args.root_path
171
@@ -214,11 +213,8 @@ def build_app(args):
213
return app
214
215
217
-async def build_server(
218
- args,
219
- llm_engine: Optional[AsyncLLMEngine] = None,
220
- **uvicorn_kwargs,
221
-) -> uvicorn.Server:
216
+async def init_app(args: Namespace,
217
+ llm_engine: Optional[AsyncLLMEngine] = None) -> FastAPI:
218
app = build_app(args)
219
220
if args.served_model_name is not None:
@@ -281,14 +277,17 @@ async def build_server(
277
)
278
app.root_path = args.root_path
279
284
- logger.info("Available routes are:")
285
- for route in app.routes:
286
- if not hasattr(route, 'methods'):
287
- continue
288
- methods = ', '.join(route.methods)
289
- logger.info("Route: %s, Methods: %s", route.path, methods)
280
+ return app
281
+
282
+
283
+async def run_server(args: Namespace,
284
+ llm_engine: Optional[AsyncLLMEngine] = None,
285
+ **uvicorn_kwargs: Any) -> None:
286
+ logger.info("vLLM API server version %s", VLLM_VERSION)
287
+ logger.info("args: %s", args)
288
291
- config = uvicorn.Config(
289
+ app = await init_app(args, llm_engine)
290
+ await serve_http(
291
app,
292
host=args.host,
293
port=args.port,
@@ -301,36 +300,6 @@ async def build_server(
300
**uvicorn_kwargs,
301
)
302
304
- return uvicorn.Server(config)
305
-
306
-
307
-async def run_server(args, llm_engine=None, **uvicorn_kwargs) -> None:
308
- logger.info("vLLM API server version %s", VLLM_VERSION)
309
- logger.info("args: %s", args)
310
-
311
- server = await build_server(
312
- args,
313
- llm_engine,
314
- **uvicorn_kwargs,
315
- )
316
-
317
- loop = asyncio.get_running_loop()
318
-
319
- server_task = loop.create_task(server.serve())
320
-
321
- def signal_handler() -> None:
322
- # prevents the uvicorn signal handler to exit early
323
- server_task.cancel()
324
-
325
- loop.add_signal_handler(signal.SIGINT, signal_handler)
326
- loop.add_signal_handler(signal.SIGTERM, signal_handler)
327
-
328
- try:
329
- await server_task
330
- except asyncio.CancelledError:
331
- print("Gracefully stopping http server")
332
- await server.shutdown()
333
-
303
304
if __name__ == "__main__":
305
# NOTE(simon):
@@ -339,4 +308,5 @@ if __name__ == "__main__":
308
description="vLLM OpenAI-Compatible RESTful API server.")
309
parser = make_arg_parser(parser)
310
args = parser.parse_args()
311
+
312
asyncio.run(run_server(args))
vllm/server/__init__.py
new
+3
@@ -0,0 +1,3 @@
1
+from .launch import serve_http
2
+
3
+__all__ = ["serve_http"]
vllm/server/launch.py
new
+42
@@ -0,0 +1,42 @@
1
+import asyncio
2
+import signal
3
+from typing import Any
4
+
5
+import uvicorn
6
+from fastapi import FastAPI
7
+
8
+from vllm.logger import init_logger
9
+
10
+logger = init_logger(__name__)
11
+
12
+
13
+async def serve_http(app: FastAPI, **uvicorn_kwargs: Any) -> None:
14
+ logger.info("Available routes are:")
15
+ for route in app.routes:
16
+ methods = getattr(route, "methods", None)
17
+ path = getattr(route, "path", None)
18
+
19
+ if methods is None or path is None:
20
+ continue
21
+
22
+ logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
23
+
24
+ config = uvicorn.Config(app, **uvicorn_kwargs)
25
+ server = uvicorn.Server(config)
26
+
27
+ loop = asyncio.get_running_loop()
28
+
29
+ server_task = loop.create_task(server.serve())
30
+
31
+ def signal_handler() -> None:
32
+ # prevents the uvicorn signal handler to exit early
33
+ server_task.cancel()
34
+
35
+ loop.add_signal_handler(signal.SIGINT, signal_handler)
36
+ loop.add_signal_handler(signal.SIGTERM, signal_handler)
37
+
38
+ try:
39
+ await server_task
40
+ except asyncio.CancelledError:
41
+ logger.info("Gracefully stopping http server")
42
+ await server.shutdown()