@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 981b0d5673
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Frontend] Factor out code for running uvicorn (#6828)

Cyrus Leung committed Jul 27, 2024 at 09:58 UTC 981b0d567355063d5453e382a85970cae083c615
4 files changed +116 -75
vllm/entrypoints/api_server.py
+50 -24
@@ -5,12 +5,12 @@ For production use, we recommend using our OpenAI compatible server.
5 We are also not going to accept PRs modifying this file, please
6 change `vllm/entrypoints/openai/api_server.py` instead.
7 """
8 -
8 +import asyncio
9 import json
10 import ssl
11 -from typing import AsyncGenerator
11 +from argparse import Namespace
12 +from typing import Any, AsyncGenerator, Optional
13
13 -import uvicorn
14 from fastapi import FastAPI, Request
15 from fastapi.responses import JSONResponse, Response, StreamingResponse
16
@@ -18,8 +18,10 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18 from vllm.engine.async_llm_engine import AsyncLLMEngine
19 from vllm.logger import init_logger
20 from vllm.sampling_params import SamplingParams
21 +from vllm.server import serve_http
22 from vllm.usage.usage_lib import UsageContext
23 from vllm.utils import FlexibleArgumentParser, random_uuid
24 +from vllm.version import __version__ as VLLM_VERSION
25
26 logger = init_logger("vllm.entrypoints.api_server")
27
@@ -81,6 +83,50 @@ async def generate(request: Request) -> Response:
83 return JSONResponse(ret)
84
85
86 +def build_app(args: Namespace) -> FastAPI:
87 + global app
88 +
89 + app.root_path = args.root_path
90 + return app
91 +
92 +
93 +async def init_app(
94 + args: Namespace,
95 + llm_engine: Optional[AsyncLLMEngine] = None,
96 +) -> FastAPI:
97 + app = build_app(args)
98 +
99 + global engine
100 +
101 + engine_args = AsyncEngineArgs.from_cli_args(args)
102 + engine = (llm_engine
103 + if llm_engine is not None else AsyncLLMEngine.from_engine_args(
104 + engine_args, usage_context=UsageContext.API_SERVER))
105 +
106 + return app
107 +
108 +
109 +async def run_server(args: Namespace,
110 + llm_engine: Optional[AsyncLLMEngine] = None,
111 + **uvicorn_kwargs: Any) -> None:
112 + logger.info("vLLM API server version %s", VLLM_VERSION)
113 + logger.info("args: %s", args)
114 +
115 + app = await init_app(args, llm_engine)
116 + await serve_http(
117 + app,
118 + host=args.host,
119 + port=args.port,
120 + log_level=args.log_level,
121 + timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
122 + ssl_keyfile=args.ssl_keyfile,
123 + ssl_certfile=args.ssl_certfile,
124 + ssl_ca_certs=args.ssl_ca_certs,
125 + ssl_cert_reqs=args.ssl_cert_reqs,
126 + **uvicorn_kwargs,
127 + )
128 +
129 +
130 if __name__ == "__main__":
131 parser = FlexibleArgumentParser()
132 parser.add_argument("--host", type=str, default=None)
@@ -105,25 +151,5 @@ if __name__ == "__main__":
151 parser.add_argument("--log-level", type=str, default="debug")
152 parser = AsyncEngineArgs.add_cli_args(parser)
153 args = parser.parse_args()
108 - engine_args = AsyncEngineArgs.from_cli_args(args)
109 - engine = AsyncLLMEngine.from_engine_args(
110 - engine_args, usage_context=UsageContext.API_SERVER)
111 -
112 - app.root_path = args.root_path
154
114 - logger.info("Available routes are:")
115 - for route in app.routes:
116 - if not hasattr(route, 'methods'):
117 - continue
118 - methods = ', '.join(route.methods)
119 - logger.info("Route: %s, Methods: %s", route.path, methods)
120 -
121 - uvicorn.run(app,
122 - host=args.host,
123 - port=args.port,
124 - log_level=args.log_level,
125 - timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
126 - ssl_keyfile=args.ssl_keyfile,
127 - ssl_certfile=args.ssl_certfile,
128 - ssl_ca_certs=args.ssl_ca_certs,
129 - ssl_cert_reqs=args.ssl_cert_reqs)
155 + asyncio.run(run_server(args))
vllm/entrypoints/openai/api_server.py
+21 -51
@@ -2,14 +2,12 @@ import asyncio
2 import importlib
3 import inspect
4 import re
5 -import signal
5 +from argparse import Namespace
6 from contextlib import asynccontextmanager
7 from http import HTTPStatus
8 -from typing import Optional, Set
8 +from typing import Any, Optional, Set
9
10 -import fastapi
11 -import uvicorn
12 -from fastapi import APIRouter, Request
10 +from fastapi import APIRouter, FastAPI, Request
11 from fastapi.exceptions import RequestValidationError
12 from fastapi.middleware.cors import CORSMiddleware
13 from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -38,6 +36,7 @@ from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
36 from vllm.entrypoints.openai.serving_tokenization import (
37 OpenAIServingTokenization)
38 from vllm.logger import init_logger
39 +from vllm.server import serve_http
40 from vllm.usage.usage_lib import UsageContext
41 from vllm.utils import FlexibleArgumentParser
42 from vllm.version import __version__ as VLLM_VERSION
@@ -57,7 +56,7 @@ _running_tasks: Set[asyncio.Task] = set()
56
57
58 @asynccontextmanager
60 -async def lifespan(app: fastapi.FastAPI):
59 +async def lifespan(app: FastAPI):
60
61 async def _force_log():
62 while True:
@@ -75,7 +74,7 @@ async def lifespan(app: fastapi.FastAPI):
74 router = APIRouter()
75
76
78 -def mount_metrics(app: fastapi.FastAPI):
77 +def mount_metrics(app: FastAPI):
78 # Add prometheus asgi middleware to route /metrics requests
79 metrics_route = Mount("/metrics", make_asgi_app())
80 # Workaround for 307 Redirect for /metrics
@@ -165,8 +164,8 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
164 return JSONResponse(content=generator.model_dump())
165
166
168 -def build_app(args):
169 - app = fastapi.FastAPI(lifespan=lifespan)
167 +def build_app(args: Namespace) -> FastAPI:
168 + app = FastAPI(lifespan=lifespan)
169 app.include_router(router)
170 app.root_path = args.root_path
171
@@ -214,11 +213,8 @@ def build_app(args):
213 return app
214
215
217 -async def build_server(
218 - args,
219 - llm_engine: Optional[AsyncLLMEngine] = None,
220 - **uvicorn_kwargs,
221 -) -> uvicorn.Server:
216 +async def init_app(args: Namespace,
217 + llm_engine: Optional[AsyncLLMEngine] = None) -> FastAPI:
218 app = build_app(args)
219
220 if args.served_model_name is not None:
@@ -281,14 +277,17 @@ async def build_server(
277 )
278 app.root_path = args.root_path
279
284 - logger.info("Available routes are:")
285 - for route in app.routes:
286 - if not hasattr(route, 'methods'):
287 - continue
288 - methods = ', '.join(route.methods)
289 - logger.info("Route: %s, Methods: %s", route.path, methods)
280 + return app
281 +
282 +
283 +async def run_server(args: Namespace,
284 + llm_engine: Optional[AsyncLLMEngine] = None,
285 + **uvicorn_kwargs: Any) -> None:
286 + logger.info("vLLM API server version %s", VLLM_VERSION)
287 + logger.info("args: %s", args)
288
291 - config = uvicorn.Config(
289 + app = await init_app(args, llm_engine)
290 + await serve_http(
291 app,
292 host=args.host,
293 port=args.port,
@@ -301,36 +300,6 @@ async def build_server(
300 **uvicorn_kwargs,
301 )
302
304 - return uvicorn.Server(config)
305 -
306 -
307 -async def run_server(args, llm_engine=None, **uvicorn_kwargs) -> None:
308 - logger.info("vLLM API server version %s", VLLM_VERSION)
309 - logger.info("args: %s", args)
310 -
311 - server = await build_server(
312 - args,
313 - llm_engine,
314 - **uvicorn_kwargs,
315 - )
316 -
317 - loop = asyncio.get_running_loop()
318 -
319 - server_task = loop.create_task(server.serve())
320 -
321 - def signal_handler() -> None:
322 - # prevents the uvicorn signal handler to exit early
323 - server_task.cancel()
324 -
325 - loop.add_signal_handler(signal.SIGINT, signal_handler)
326 - loop.add_signal_handler(signal.SIGTERM, signal_handler)
327 -
328 - try:
329 - await server_task
330 - except asyncio.CancelledError:
331 - print("Gracefully stopping http server")
332 - await server.shutdown()
333 -
303
304 if __name__ == "__main__":
305 # NOTE(simon):
@@ -339,4 +308,5 @@ if __name__ == "__main__":
308 description="vLLM OpenAI-Compatible RESTful API server.")
309 parser = make_arg_parser(parser)
310 args = parser.parse_args()
311 +
312 asyncio.run(run_server(args))
vllm/server/__init__.py new
+3
@@ -0,0 +1,3 @@
1 +from .launch import serve_http
2 +
3 +__all__ = ["serve_http"]
vllm/server/launch.py new
+42
@@ -0,0 +1,42 @@
1 +import asyncio
2 +import signal
3 +from typing import Any
4 +
5 +import uvicorn
6 +from fastapi import FastAPI
7 +
8 +from vllm.logger import init_logger
9 +
10 +logger = init_logger(__name__)
11 +
12 +
13 +async def serve_http(app: FastAPI, **uvicorn_kwargs: Any) -> None:
14 + logger.info("Available routes are:")
15 + for route in app.routes:
16 + methods = getattr(route, "methods", None)
17 + path = getattr(route, "path", None)
18 +
19 + if methods is None or path is None:
20 + continue
21 +
22 + logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
23 +
24 + config = uvicorn.Config(app, **uvicorn_kwargs)
25 + server = uvicorn.Server(config)
26 +
27 + loop = asyncio.get_running_loop()
28 +
29 + server_task = loop.create_task(server.serve())
30 +
31 + def signal_handler() -> None:
32 + # prevents the uvicorn signal handler to exit early
33 + server_task.cancel()
34 +
35 + loop.add_signal_handler(signal.SIGINT, signal_handler)
36 + loop.add_signal_handler(signal.SIGTERM, signal_handler)
37 +
38 + try:
39 + await server_task
40 + except asyncio.CancelledError:
41 + logger.info("Gracefully stopping http server")
42 + await server.shutdown()