@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7eb0cb4a14
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

Revert "[Frontend] Factor out code for running uvicorn" (#7012)

Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>

Simon Mo committed Jul 31, 2024 at 16:34 UTC 7eb0cb4a14ff3de84bf18fad8054d12ea8000c22
5 files changed +75 -117
pyproject.toml
-1
@@ -60,7 +60,6 @@ files = [
60 "vllm/logging",
61 "vllm/multimodal",
62 "vllm/platforms",
63 - "vllm/server",
63 "vllm/transformers_utils",
64 "vllm/triton_utils",
65 "vllm/usage",
vllm/entrypoints/api_server.py
+24 -50
@@ -5,12 +5,12 @@ For production use, we recommend using our OpenAI compatible server.
5 We are also not going to accept PRs modifying this file, please
6 change `vllm/entrypoints/openai/api_server.py` instead.
7 """
8 -import asyncio
8 +
9 import json
10 import ssl
11 -from argparse import Namespace
12 -from typing import Any, AsyncGenerator, Optional
11 +from typing import AsyncGenerator
12
13 +import uvicorn
14 from fastapi import FastAPI, Request
15 from fastapi.responses import JSONResponse, Response, StreamingResponse
16
@@ -18,10 +18,8 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18 from vllm.engine.async_llm_engine import AsyncLLMEngine
19 from vllm.logger import init_logger
20 from vllm.sampling_params import SamplingParams
21 -from vllm.server import serve_http
21 from vllm.usage.usage_lib import UsageContext
22 from vllm.utils import FlexibleArgumentParser, random_uuid
24 -from vllm.version import __version__ as VLLM_VERSION
23
24 logger = init_logger("vllm.entrypoints.api_server")
25
@@ -83,50 +81,6 @@ async def generate(request: Request) -> Response:
81 return JSONResponse(ret)
82
83
86 -def build_app(args: Namespace) -> FastAPI:
87 - global app
88 -
89 - app.root_path = args.root_path
90 - return app
91 -
92 -
93 -async def init_app(
94 - args: Namespace,
95 - llm_engine: Optional[AsyncLLMEngine] = None,
96 -) -> FastAPI:
97 - app = build_app(args)
98 -
99 - global engine
100 -
101 - engine_args = AsyncEngineArgs.from_cli_args(args)
102 - engine = (llm_engine
103 - if llm_engine is not None else AsyncLLMEngine.from_engine_args(
104 - engine_args, usage_context=UsageContext.API_SERVER))
105 -
106 - return app
107 -
108 -
109 -async def run_server(args: Namespace,
110 - llm_engine: Optional[AsyncLLMEngine] = None,
111 - **uvicorn_kwargs: Any) -> None:
112 - logger.info("vLLM API server version %s", VLLM_VERSION)
113 - logger.info("args: %s", args)
114 -
115 - app = await init_app(args, llm_engine)
116 - await serve_http(
117 - app,
118 - host=args.host,
119 - port=args.port,
120 - log_level=args.log_level,
121 - timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
122 - ssl_keyfile=args.ssl_keyfile,
123 - ssl_certfile=args.ssl_certfile,
124 - ssl_ca_certs=args.ssl_ca_certs,
125 - ssl_cert_reqs=args.ssl_cert_reqs,
126 - **uvicorn_kwargs,
127 - )
128 -
129 -
84 if __name__ == "__main__":
85 parser = FlexibleArgumentParser()
86 parser.add_argument("--host", type=str, default=None)
@@ -151,5 +105,25 @@ if __name__ == "__main__":
105 parser.add_argument("--log-level", type=str, default="debug")
106 parser = AsyncEngineArgs.add_cli_args(parser)
107 args = parser.parse_args()
108 + engine_args = AsyncEngineArgs.from_cli_args(args)
109 + engine = AsyncLLMEngine.from_engine_args(
110 + engine_args, usage_context=UsageContext.API_SERVER)
111 +
112 + app.root_path = args.root_path
113
155 - asyncio.run(run_server(args))
114 + logger.info("Available routes are:")
115 + for route in app.routes:
116 + if not hasattr(route, 'methods'):
117 + continue
118 + methods = ', '.join(route.methods)
119 + logger.info("Route: %s, Methods: %s", route.path, methods)
120 +
121 + uvicorn.run(app,
122 + host=args.host,
123 + port=args.port,
124 + log_level=args.log_level,
125 + timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
126 + ssl_keyfile=args.ssl_keyfile,
127 + ssl_certfile=args.ssl_certfile,
128 + ssl_ca_certs=args.ssl_ca_certs,
129 + ssl_cert_reqs=args.ssl_cert_reqs)
vllm/entrypoints/openai/api_server.py
+51 -21
@@ -2,12 +2,14 @@ import asyncio
2 import importlib
3 import inspect
4 import re
5 -from argparse import Namespace
5 +import signal
6 from contextlib import asynccontextmanager
7 from http import HTTPStatus
8 -from typing import Any, Optional, Set
8 +from typing import Optional, Set
9
10 -from fastapi import APIRouter, FastAPI, Request
10 +import fastapi
11 +import uvicorn
12 +from fastapi import APIRouter, Request
13 from fastapi.exceptions import RequestValidationError
14 from fastapi.middleware.cors import CORSMiddleware
15 from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -36,7 +38,6 @@ from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
38 from vllm.entrypoints.openai.serving_tokenization import (
39 OpenAIServingTokenization)
40 from vllm.logger import init_logger
39 -from vllm.server import serve_http
41 from vllm.usage.usage_lib import UsageContext
42 from vllm.utils import FlexibleArgumentParser
43 from vllm.version import __version__ as VLLM_VERSION
@@ -56,7 +57,7 @@ _running_tasks: Set[asyncio.Task] = set()
57
58
59 @asynccontextmanager
59 -async def lifespan(app: FastAPI):
60 +async def lifespan(app: fastapi.FastAPI):
61
62 async def _force_log():
63 while True:
@@ -74,7 +75,7 @@ async def lifespan(app: FastAPI):
75 router = APIRouter()
76
77
77 -def mount_metrics(app: FastAPI):
78 +def mount_metrics(app: fastapi.FastAPI):
79 # Add prometheus asgi middleware to route /metrics requests
80 metrics_route = Mount("/metrics", make_asgi_app())
81 # Workaround for 307 Redirect for /metrics
@@ -164,8 +165,8 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
165 return JSONResponse(content=generator.model_dump())
166
167
167 -def build_app(args: Namespace) -> FastAPI:
168 - app = FastAPI(lifespan=lifespan)
168 +def build_app(args):
169 + app = fastapi.FastAPI(lifespan=lifespan)
170 app.include_router(router)
171 app.root_path = args.root_path
172
@@ -213,8 +214,11 @@ def build_app(args: Namespace) -> FastAPI:
214 return app
215
216
216 -async def init_app(args: Namespace,
217 - llm_engine: Optional[AsyncLLMEngine] = None) -> FastAPI:
217 +async def build_server(
218 + args,
219 + llm_engine: Optional[AsyncLLMEngine] = None,
220 + **uvicorn_kwargs,
221 +) -> uvicorn.Server:
222 app = build_app(args)
223
224 if args.served_model_name is not None:
@@ -277,17 +281,14 @@ async def init_app(args: Namespace,
281 )
282 app.root_path = args.root_path
283
280 - return app
281 -
282 -
283 -async def run_server(args: Namespace,
284 - llm_engine: Optional[AsyncLLMEngine] = None,
285 - **uvicorn_kwargs: Any) -> None:
286 - logger.info("vLLM API server version %s", VLLM_VERSION)
287 - logger.info("args: %s", args)
284 + logger.info("Available routes are:")
285 + for route in app.routes:
286 + if not hasattr(route, 'methods'):
287 + continue
288 + methods = ', '.join(route.methods)
289 + logger.info("Route: %s, Methods: %s", route.path, methods)
290
289 - app = await init_app(args, llm_engine)
290 - await serve_http(
291 + config = uvicorn.Config(
292 app,
293 host=args.host,
294 port=args.port,
@@ -300,6 +301,36 @@ async def run_server(args: Namespace,
301 **uvicorn_kwargs,
302 )
303
304 + return uvicorn.Server(config)
305 +
306 +
307 +async def run_server(args, llm_engine=None, **uvicorn_kwargs) -> None:
308 + logger.info("vLLM API server version %s", VLLM_VERSION)
309 + logger.info("args: %s", args)
310 +
311 + server = await build_server(
312 + args,
313 + llm_engine,
314 + **uvicorn_kwargs,
315 + )
316 +
317 + loop = asyncio.get_running_loop()
318 +
319 + server_task = loop.create_task(server.serve())
320 +
321 + def signal_handler() -> None:
322 + # prevents the uvicorn signal handler to exit early
323 + server_task.cancel()
324 +
325 + loop.add_signal_handler(signal.SIGINT, signal_handler)
326 + loop.add_signal_handler(signal.SIGTERM, signal_handler)
327 +
328 + try:
329 + await server_task
330 + except asyncio.CancelledError:
331 + print("Gracefully stopping http server")
332 + await server.shutdown()
333 +
334
335 if __name__ == "__main__":
336 # NOTE(simon):
@@ -308,5 +339,4 @@ if __name__ == "__main__":
339 description="vLLM OpenAI-Compatible RESTful API server.")
340 parser = make_arg_parser(parser)
341 args = parser.parse_args()
311 -
342 asyncio.run(run_server(args))
vllm/server/__init__.py deleted
-3
@@ -1,3 +0,0 @@
1 -from .launch import serve_http
2 -
3 -__all__ = ["serve_http"]
vllm/server/launch.py deleted
-42
@@ -1,42 +0,0 @@
1 -import asyncio
2 -import signal
3 -from typing import Any
4 -
5 -import uvicorn
6 -from fastapi import FastAPI
7 -
8 -from vllm.logger import init_logger
9 -
10 -logger = init_logger(__name__)
11 -
12 -
13 -async def serve_http(app: FastAPI, **uvicorn_kwargs: Any) -> None:
14 - logger.info("Available routes are:")
15 - for route in app.routes:
16 - methods = getattr(route, "methods", None)
17 - path = getattr(route, "path", None)
18 -
19 - if methods is None or path is None:
20 - continue
21 -
22 - logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
23 -
24 - config = uvicorn.Config(app, **uvicorn_kwargs)
25 - server = uvicorn.Server(config)
26 -
27 - loop = asyncio.get_running_loop()
28 -
29 - server_task = loop.create_task(server.serve())
30 -
31 - def signal_handler() -> None:
32 - # prevents the uvicorn signal handler to exit early
33 - server_task.cancel()
34 -
35 - loop.add_signal_handler(signal.SIGINT, signal_handler)
36 - loop.add_signal_handler(signal.SIGTERM, signal_handler)
37 -
38 - try:
39 - await server_task
40 - except asyncio.CancelledError:
41 - logger.info("Gracefully stopping http server")
42 - await server.shutdown()