A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
Revert "[Frontend] Factor out code for running uvicorn" (#7012)
Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>
Simon Mo committed
Jul 31, 2024 at 16:34 UTC
7eb0cb4a14ff3de84bf18fad8054d12ea8000c22
5 files changed
+75
-117
pyproject.toml
-1
@@ -60,7 +60,6 @@ files = [
60
"vllm/logging",
61
"vllm/multimodal",
62
"vllm/platforms",
63
- "vllm/server",
63
"vllm/transformers_utils",
64
"vllm/triton_utils",
65
"vllm/usage",
vllm/entrypoints/api_server.py
+24
-50
@@ -5,12 +5,12 @@ For production use, we recommend using our OpenAI compatible server.
5
We are also not going to accept PRs modifying this file, please
6
change `vllm/entrypoints/openai/api_server.py` instead.
7
"""
8
-import asyncio
8
+
9
import json
10
import ssl
11
-from argparse import Namespace
12
-from typing import Any, AsyncGenerator, Optional
11
+from typing import AsyncGenerator
12
13
+import uvicorn
14
from fastapi import FastAPI, Request
15
from fastapi.responses import JSONResponse, Response, StreamingResponse
16
@@ -18,10 +18,8 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.async_llm_engine import AsyncLLMEngine
19
from vllm.logger import init_logger
20
from vllm.sampling_params import SamplingParams
21
-from vllm.server import serve_http
21
from vllm.usage.usage_lib import UsageContext
22
from vllm.utils import FlexibleArgumentParser, random_uuid
24
-from vllm.version import __version__ as VLLM_VERSION
23
24
logger = init_logger("vllm.entrypoints.api_server")
25
@@ -83,50 +81,6 @@ async def generate(request: Request) -> Response:
81
return JSONResponse(ret)
82
83
86
-def build_app(args: Namespace) -> FastAPI:
87
- global app
88
-
89
- app.root_path = args.root_path
90
- return app
91
-
92
-
93
-async def init_app(
94
- args: Namespace,
95
- llm_engine: Optional[AsyncLLMEngine] = None,
96
-) -> FastAPI:
97
- app = build_app(args)
98
-
99
- global engine
100
-
101
- engine_args = AsyncEngineArgs.from_cli_args(args)
102
- engine = (llm_engine
103
- if llm_engine is not None else AsyncLLMEngine.from_engine_args(
104
- engine_args, usage_context=UsageContext.API_SERVER))
105
-
106
- return app
107
-
108
-
109
-async def run_server(args: Namespace,
110
- llm_engine: Optional[AsyncLLMEngine] = None,
111
- **uvicorn_kwargs: Any) -> None:
112
- logger.info("vLLM API server version %s", VLLM_VERSION)
113
- logger.info("args: %s", args)
114
-
115
- app = await init_app(args, llm_engine)
116
- await serve_http(
117
- app,
118
- host=args.host,
119
- port=args.port,
120
- log_level=args.log_level,
121
- timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
122
- ssl_keyfile=args.ssl_keyfile,
123
- ssl_certfile=args.ssl_certfile,
124
- ssl_ca_certs=args.ssl_ca_certs,
125
- ssl_cert_reqs=args.ssl_cert_reqs,
126
- **uvicorn_kwargs,
127
- )
128
-
129
-
84
if __name__ == "__main__":
85
parser = FlexibleArgumentParser()
86
parser.add_argument("--host", type=str, default=None)
@@ -151,5 +105,25 @@ if __name__ == "__main__":
105
parser.add_argument("--log-level", type=str, default="debug")
106
parser = AsyncEngineArgs.add_cli_args(parser)
107
args = parser.parse_args()
108
+ engine_args = AsyncEngineArgs.from_cli_args(args)
109
+ engine = AsyncLLMEngine.from_engine_args(
110
+ engine_args, usage_context=UsageContext.API_SERVER)
111
+
112
+ app.root_path = args.root_path
113
155
- asyncio.run(run_server(args))
114
+ logger.info("Available routes are:")
115
+ for route in app.routes:
116
+ if not hasattr(route, 'methods'):
117
+ continue
118
+ methods = ', '.join(route.methods)
119
+ logger.info("Route: %s, Methods: %s", route.path, methods)
120
+
121
+ uvicorn.run(app,
122
+ host=args.host,
123
+ port=args.port,
124
+ log_level=args.log_level,
125
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
126
+ ssl_keyfile=args.ssl_keyfile,
127
+ ssl_certfile=args.ssl_certfile,
128
+ ssl_ca_certs=args.ssl_ca_certs,
129
+ ssl_cert_reqs=args.ssl_cert_reqs)
vllm/entrypoints/openai/api_server.py
+51
-21
@@ -2,12 +2,14 @@ import asyncio
2
import importlib
3
import inspect
4
import re
5
-from argparse import Namespace
5
+import signal
6
from contextlib import asynccontextmanager
7
from http import HTTPStatus
8
-from typing import Any, Optional, Set
8
+from typing import Optional, Set
9
10
-from fastapi import APIRouter, FastAPI, Request
10
+import fastapi
11
+import uvicorn
12
+from fastapi import APIRouter, Request
13
from fastapi.exceptions import RequestValidationError
14
from fastapi.middleware.cors import CORSMiddleware
15
from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -36,7 +38,6 @@ from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
38
from vllm.entrypoints.openai.serving_tokenization import (
39
OpenAIServingTokenization)
40
from vllm.logger import init_logger
39
-from vllm.server import serve_http
41
from vllm.usage.usage_lib import UsageContext
42
from vllm.utils import FlexibleArgumentParser
43
from vllm.version import __version__ as VLLM_VERSION
@@ -56,7 +57,7 @@ _running_tasks: Set[asyncio.Task] = set()
57
58
59
@asynccontextmanager
59
-async def lifespan(app: FastAPI):
60
+async def lifespan(app: fastapi.FastAPI):
61
62
async def _force_log():
63
while True:
@@ -74,7 +75,7 @@ async def lifespan(app: FastAPI):
75
router = APIRouter()
76
77
77
-def mount_metrics(app: FastAPI):
78
+def mount_metrics(app: fastapi.FastAPI):
79
# Add prometheus asgi middleware to route /metrics requests
80
metrics_route = Mount("/metrics", make_asgi_app())
81
# Workaround for 307 Redirect for /metrics
@@ -164,8 +165,8 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
165
return JSONResponse(content=generator.model_dump())
166
167
167
-def build_app(args: Namespace) -> FastAPI:
168
- app = FastAPI(lifespan=lifespan)
168
+def build_app(args):
169
+ app = fastapi.FastAPI(lifespan=lifespan)
170
app.include_router(router)
171
app.root_path = args.root_path
172
@@ -213,8 +214,11 @@ def build_app(args: Namespace) -> FastAPI:
214
return app
215
216
216
-async def init_app(args: Namespace,
217
- llm_engine: Optional[AsyncLLMEngine] = None) -> FastAPI:
217
+async def build_server(
218
+ args,
219
+ llm_engine: Optional[AsyncLLMEngine] = None,
220
+ **uvicorn_kwargs,
221
+) -> uvicorn.Server:
222
app = build_app(args)
223
224
if args.served_model_name is not None:
@@ -277,17 +281,14 @@ async def init_app(args: Namespace,
281
)
282
app.root_path = args.root_path
283
280
- return app
281
-
282
-
283
-async def run_server(args: Namespace,
284
- llm_engine: Optional[AsyncLLMEngine] = None,
285
- **uvicorn_kwargs: Any) -> None:
286
- logger.info("vLLM API server version %s", VLLM_VERSION)
287
- logger.info("args: %s", args)
284
+ logger.info("Available routes are:")
285
+ for route in app.routes:
286
+ if not hasattr(route, 'methods'):
287
+ continue
288
+ methods = ', '.join(route.methods)
289
+ logger.info("Route: %s, Methods: %s", route.path, methods)
290
289
- app = await init_app(args, llm_engine)
290
- await serve_http(
291
+ config = uvicorn.Config(
292
app,
293
host=args.host,
294
port=args.port,
@@ -300,6 +301,36 @@ async def run_server(args: Namespace,
301
**uvicorn_kwargs,
302
)
303
304
+ return uvicorn.Server(config)
305
+
306
+
307
+async def run_server(args, llm_engine=None, **uvicorn_kwargs) -> None:
308
+ logger.info("vLLM API server version %s", VLLM_VERSION)
309
+ logger.info("args: %s", args)
310
+
311
+ server = await build_server(
312
+ args,
313
+ llm_engine,
314
+ **uvicorn_kwargs,
315
+ )
316
+
317
+ loop = asyncio.get_running_loop()
318
+
319
+ server_task = loop.create_task(server.serve())
320
+
321
+ def signal_handler() -> None:
322
+ # prevents the uvicorn signal handler to exit early
323
+ server_task.cancel()
324
+
325
+ loop.add_signal_handler(signal.SIGINT, signal_handler)
326
+ loop.add_signal_handler(signal.SIGTERM, signal_handler)
327
+
328
+ try:
329
+ await server_task
330
+ except asyncio.CancelledError:
331
+ print("Gracefully stopping http server")
332
+ await server.shutdown()
333
+
334
335
if __name__ == "__main__":
336
# NOTE(simon):
@@ -308,5 +339,4 @@ if __name__ == "__main__":
339
description="vLLM OpenAI-Compatible RESTful API server.")
340
parser = make_arg_parser(parser)
341
args = parser.parse_args()
311
-
342
asyncio.run(run_server(args))
vllm/server/__init__.py
deleted
-3
@@ -1,3 +0,0 @@
1
-from .launch import serve_http
2
-
3
-__all__ = ["serve_http"]
vllm/server/launch.py
deleted
-42
@@ -1,42 +0,0 @@
1
-import asyncio
2
-import signal
3
-from typing import Any
4
-
5
-import uvicorn
6
-from fastapi import FastAPI
7
-
8
-from vllm.logger import init_logger
9
-
10
-logger = init_logger(__name__)
11
-
12
-
13
-async def serve_http(app: FastAPI, **uvicorn_kwargs: Any) -> None:
14
- logger.info("Available routes are:")
15
- for route in app.routes:
16
- methods = getattr(route, "methods", None)
17
- path = getattr(route, "path", None)
18
-
19
- if methods is None or path is None:
20
- continue
21
-
22
- logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
23
-
24
- config = uvicorn.Config(app, **uvicorn_kwargs)
25
- server = uvicorn.Server(config)
26
-
27
- loop = asyncio.get_running_loop()
28
-
29
- server_task = loop.create_task(server.serve())
30
-
31
- def signal_handler() -> None:
32
- # prevents the uvicorn signal handler to exit early
33
- server_task.cancel()
34
-
35
- loop.add_signal_handler(signal.SIGINT, signal_handler)
36
- loop.add_signal_handler(signal.SIGTERM, signal_handler)
37
-
38
- try:
39
- await server_task
40
- except asyncio.CancelledError:
41
- logger.info("Gracefully stopping http server")
42
- await server.shutdown()