A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Frontend] Reapply "Factor out code for running uvicorn" (#7095)
Cyrus Leung committed
Aug 5, 2024 at 11:40 UTC
cc08fc7225616aeb6709a2e75e5ac47ace124985
3 files changed
+125
-82
vllm/entrypoints/api_server.py
+53
-24
@@ -5,21 +5,23 @@ For production use, we recommend using our OpenAI compatible server.
5
We are also not going to accept PRs modifying this file, please
6
change `vllm/entrypoints/openai/api_server.py` instead.
7
"""
8
-
8
+import asyncio
9
import json
10
import ssl
11
-from typing import AsyncGenerator
11
+from argparse import Namespace
12
+from typing import Any, AsyncGenerator, Optional
13
13
-import uvicorn
14
from fastapi import FastAPI, Request
15
from fastapi.responses import JSONResponse, Response, StreamingResponse
16
17
from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.async_llm_engine import AsyncLLMEngine
19
+from vllm.entrypoints.launcher import serve_http
20
from vllm.logger import init_logger
21
from vllm.sampling_params import SamplingParams
22
from vllm.usage.usage_lib import UsageContext
23
from vllm.utils import FlexibleArgumentParser, random_uuid
24
+from vllm.version import __version__ as VLLM_VERSION
25
26
logger = init_logger("vllm.entrypoints.api_server")
27
@@ -81,6 +83,53 @@ async def generate(request: Request) -> Response:
83
return JSONResponse(ret)
84
85
86
+def build_app(args: Namespace) -> FastAPI:
87
+ global app
88
+
89
+ app.root_path = args.root_path
90
+ return app
91
+
92
+
93
+async def init_app(
94
+ args: Namespace,
95
+ llm_engine: Optional[AsyncLLMEngine] = None,
96
+) -> FastAPI:
97
+ app = build_app(args)
98
+
99
+ global engine
100
+
101
+ engine_args = AsyncEngineArgs.from_cli_args(args)
102
+ engine = (llm_engine
103
+ if llm_engine is not None else AsyncLLMEngine.from_engine_args(
104
+ engine_args, usage_context=UsageContext.API_SERVER))
105
+
106
+ return app
107
+
108
+
109
+async def run_server(args: Namespace,
110
+ llm_engine: Optional[AsyncLLMEngine] = None,
111
+ **uvicorn_kwargs: Any) -> None:
112
+ logger.info("vLLM API server version %s", VLLM_VERSION)
113
+ logger.info("args: %s", args)
114
+
115
+ app = await init_app(args, llm_engine)
116
+
117
+ shutdown_task = await serve_http(
118
+ app,
119
+ host=args.host,
120
+ port=args.port,
121
+ log_level=args.log_level,
122
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
123
+ ssl_keyfile=args.ssl_keyfile,
124
+ ssl_certfile=args.ssl_certfile,
125
+ ssl_ca_certs=args.ssl_ca_certs,
126
+ ssl_cert_reqs=args.ssl_cert_reqs,
127
+ **uvicorn_kwargs,
128
+ )
129
+
130
+ await shutdown_task
131
+
132
+
133
if __name__ == "__main__":
134
parser = FlexibleArgumentParser()
135
parser.add_argument("--host", type=str, default=None)
@@ -105,25 +154,5 @@ if __name__ == "__main__":
154
parser.add_argument("--log-level", type=str, default="debug")
155
parser = AsyncEngineArgs.add_cli_args(parser)
156
args = parser.parse_args()
108
- engine_args = AsyncEngineArgs.from_cli_args(args)
109
- engine = AsyncLLMEngine.from_engine_args(
110
- engine_args, usage_context=UsageContext.API_SERVER)
111
-
112
- app.root_path = args.root_path
157
114
- logger.info("Available routes are:")
115
- for route in app.routes:
116
- if not hasattr(route, 'methods'):
117
- continue
118
- methods = ', '.join(route.methods)
119
- logger.info("Route: %s, Methods: %s", route.path, methods)
120
-
121
- uvicorn.run(app,
122
- host=args.host,
123
- port=args.port,
124
- log_level=args.log_level,
125
- timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
126
- ssl_keyfile=args.ssl_keyfile,
127
- ssl_certfile=args.ssl_certfile,
128
- ssl_ca_certs=args.ssl_ca_certs,
129
- ssl_cert_reqs=args.ssl_cert_reqs)
158
+ asyncio.run(run_server(args))
vllm/entrypoints/launcher.py
new
+46
@@ -0,0 +1,46 @@
1
+import asyncio
2
+import signal
3
+from typing import Any
4
+
5
+import uvicorn
6
+from fastapi import FastAPI
7
+
8
+from vllm.logger import init_logger
9
+
10
+logger = init_logger(__name__)
11
+
12
+
13
+async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
14
+ logger.info("Available routes are:")
15
+ for route in app.routes:
16
+ methods = getattr(route, "methods", None)
17
+ path = getattr(route, "path", None)
18
+
19
+ if methods is None or path is None:
20
+ continue
21
+
22
+ logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
23
+
24
+ config = uvicorn.Config(app, **uvicorn_kwargs)
25
+ server = uvicorn.Server(config)
26
+
27
+ loop = asyncio.get_running_loop()
28
+
29
+ server_task = loop.create_task(server.serve())
30
+
31
+ def signal_handler() -> None:
32
+ # prevents the uvicorn signal handler to exit early
33
+ server_task.cancel()
34
+
35
+ async def dummy_shutdown() -> None:
36
+ pass
37
+
38
+ loop.add_signal_handler(signal.SIGINT, signal_handler)
39
+ loop.add_signal_handler(signal.SIGTERM, signal_handler)
40
+
41
+ try:
42
+ await server_task
43
+ return dummy_shutdown()
44
+ except asyncio.CancelledError:
45
+ logger.info("Gracefully stopping http server")
46
+ return server.shutdown()
vllm/entrypoints/openai/api_server.py
+26
-58
@@ -2,15 +2,13 @@ import asyncio
2
import importlib
3
import inspect
4
import re
5
-import signal
5
+from argparse import Namespace
6
from contextlib import asynccontextmanager
7
from http import HTTPStatus
8
from multiprocessing import Process
9
from typing import AsyncIterator, Set
10
11
-import fastapi
12
-import uvicorn
13
-from fastapi import APIRouter, Request
11
+from fastapi import APIRouter, FastAPI, Request
12
from fastapi.exceptions import RequestValidationError
13
from fastapi.middleware.cors import CORSMiddleware
14
from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -22,6 +20,7 @@ from vllm.config import ModelConfig
20
from vllm.engine.arg_utils import AsyncEngineArgs
21
from vllm.engine.async_llm_engine import AsyncLLMEngine
22
from vllm.engine.protocol import AsyncEngineClient
23
+from vllm.entrypoints.launcher import serve_http
24
from vllm.entrypoints.logger import RequestLogger
25
from vllm.entrypoints.openai.cli_args import make_arg_parser
26
# yapf conflicts with isort for this block
@@ -71,7 +70,7 @@ def model_is_embedding(model_name: str) -> bool:
70
71
72
@asynccontextmanager
74
-async def lifespan(app: fastapi.FastAPI):
73
+async def lifespan(app: FastAPI):
74
75
async def _force_log():
76
while True:
@@ -135,7 +134,7 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
134
router = APIRouter()
135
136
138
-def mount_metrics(app: fastapi.FastAPI):
137
+def mount_metrics(app: FastAPI):
138
# Add prometheus asgi middleware to route /metrics requests
139
metrics_route = Mount("/metrics", make_asgi_app())
140
# Workaround for 307 Redirect for /metrics
@@ -225,8 +224,8 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
224
return JSONResponse(content=generator.model_dump())
225
226
228
-def build_app(args):
229
- app = fastapi.FastAPI(lifespan=lifespan)
227
+def build_app(args: Namespace) -> FastAPI:
228
+ app = FastAPI(lifespan=lifespan)
229
app.include_router(router)
230
app.root_path = args.root_path
231
@@ -274,11 +273,10 @@ def build_app(args):
273
return app
274
275
277
-async def build_server(
276
+async def init_app(
277
async_engine_client: AsyncEngineClient,
279
- args,
280
- **uvicorn_kwargs,
281
-) -> uvicorn.Server:
278
+ args: Namespace,
279
+) -> FastAPI:
280
app = build_app(args)
281
282
if args.served_model_name is not None:
@@ -334,62 +332,31 @@ async def build_server(
332
)
333
app.root_path = args.root_path
334
337
- logger.info("Available routes are:")
338
- for route in app.routes:
339
- if not hasattr(route, 'methods'):
340
- continue
341
- methods = ', '.join(route.methods)
342
- logger.info("Route: %s, Methods: %s", route.path, methods)
343
-
344
- config = uvicorn.Config(
345
- app,
346
- host=args.host,
347
- port=args.port,
348
- log_level=args.uvicorn_log_level,
349
- timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
350
- ssl_keyfile=args.ssl_keyfile,
351
- ssl_certfile=args.ssl_certfile,
352
- ssl_ca_certs=args.ssl_ca_certs,
353
- ssl_cert_reqs=args.ssl_cert_reqs,
354
- **uvicorn_kwargs,
355
- )
356
-
357
- return uvicorn.Server(config)
335
+ return app
336
337
338
async def run_server(args, **uvicorn_kwargs) -> None:
339
logger.info("vLLM API server version %s", VLLM_VERSION)
340
logger.info("args: %s", args)
341
364
- shutdown_task = None
342
async with build_async_engine_client(args) as async_engine_client:
366
-
367
- server = await build_server(
368
- async_engine_client,
369
- args,
343
+ app = await init_app(async_engine_client, args)
344
+
345
+ shutdown_task = await serve_http(
346
+ app,
347
+ host=args.host,
348
+ port=args.port,
349
+ log_level=args.uvicorn_log_level,
350
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
351
+ ssl_keyfile=args.ssl_keyfile,
352
+ ssl_certfile=args.ssl_certfile,
353
+ ssl_ca_certs=args.ssl_ca_certs,
354
+ ssl_cert_reqs=args.ssl_cert_reqs,
355
**uvicorn_kwargs,
356
)
357
373
- loop = asyncio.get_running_loop()
374
-
375
- server_task = loop.create_task(server.serve())
376
-
377
- def signal_handler() -> None:
378
- # prevents the uvicorn signal handler to exit early
379
- server_task.cancel()
380
-
381
- loop.add_signal_handler(signal.SIGINT, signal_handler)
382
- loop.add_signal_handler(signal.SIGTERM, signal_handler)
383
-
384
- try:
385
- await server_task
386
- except asyncio.CancelledError:
387
- logger.info("Gracefully stopping http server")
388
- shutdown_task = server.shutdown()
389
-
390
- if shutdown_task:
391
- # NB: Await server shutdown only after the backend context is exited
392
- await shutdown_task
358
+ # NB: Await server shutdown only after the backend context is exited
359
+ await shutdown_task
360
361
362
if __name__ == "__main__":
@@ -399,4 +366,5 @@ if __name__ == "__main__":
366
description="vLLM OpenAI-Compatible RESTful API server.")
367
parser = make_arg_parser(parser)
368
args = parser.parse_args()
369
+
370
asyncio.run(run_server(args))