A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Frontend] split run_server into build_server and run_server (#6740)
Daniele committed
Jul 24, 2024 at 19:36 UTC
ee812580f7ae3969cf2550cc4dab31490bfea950
2 files changed
+52
-28
vllm/entrypoints/openai/api_server.py
+50
-27
@@ -2,6 +2,7 @@ import asyncio
2
import importlib
3
import inspect
4
import re
5
+import signal
6
from contextlib import asynccontextmanager
7
from http import HTTPStatus
8
from typing import Optional, Set
@@ -213,12 +214,13 @@ def build_app(args):
214
return app
215
216
216
-def run_server(args, llm_engine=None):
217
+async def build_server(
218
+ args,
219
+ llm_engine: Optional[AsyncLLMEngine] = None,
220
+ **uvicorn_kwargs,
221
+) -> uvicorn.Server:
222
app = build_app(args)
223
219
- logger.info("vLLM API server version %s", VLLM_VERSION)
220
- logger.info("args: %s", args)
221
-
224
if args.served_model_name is not None:
225
served_model_names = args.served_model_name
226
else:
@@ -231,19 +233,7 @@ def run_server(args, llm_engine=None):
233
if llm_engine is not None else AsyncLLMEngine.from_engine_args(
234
engine_args, usage_context=UsageContext.OPENAI_API_SERVER))
235
234
- event_loop: Optional[asyncio.AbstractEventLoop]
235
- try:
236
- event_loop = asyncio.get_running_loop()
237
- except RuntimeError:
238
- event_loop = None
239
-
240
- if event_loop is not None and event_loop.is_running():
241
- # If the current is instanced by Ray Serve,
242
- # there is already a running event loop
243
- model_config = event_loop.run_until_complete(engine.get_model_config())
244
- else:
245
- # When using single vLLM without engine_use_ray
246
- model_config = asyncio.run(engine.get_model_config())
236
+ model_config = await engine.get_model_config()
237
238
if args.disable_log_requests:
239
request_logger = None
@@ -296,15 +286,48 @@ def run_server(args, llm_engine=None):
286
methods = ', '.join(route.methods)
287
logger.info("Route: %s, Methods: %s", route.path, methods)
288
299
- uvicorn.run(app,
300
- host=args.host,
301
- port=args.port,
302
- log_level=args.uvicorn_log_level,
303
- timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
304
- ssl_keyfile=args.ssl_keyfile,
305
- ssl_certfile=args.ssl_certfile,
306
- ssl_ca_certs=args.ssl_ca_certs,
307
- ssl_cert_reqs=args.ssl_cert_reqs)
289
+ config = uvicorn.Config(
290
+ app,
291
+ host=args.host,
292
+ port=args.port,
293
+ log_level=args.uvicorn_log_level,
294
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
295
+ ssl_keyfile=args.ssl_keyfile,
296
+ ssl_certfile=args.ssl_certfile,
297
+ ssl_ca_certs=args.ssl_ca_certs,
298
+ ssl_cert_reqs=args.ssl_cert_reqs,
299
+ **uvicorn_kwargs,
300
+ )
301
+
302
+ return uvicorn.Server(config)
303
+
304
+
305
+async def run_server(args, llm_engine=None, **uvicorn_kwargs) -> None:
306
+ logger.info("vLLM API server version %s", VLLM_VERSION)
307
+ logger.info("args: %s", args)
308
+
309
+ server = await build_server(
310
+ args,
311
+ llm_engine,
312
+ **uvicorn_kwargs,
313
+ )
314
+
315
+ loop = asyncio.get_running_loop()
316
+
317
+ server_task = loop.create_task(server.serve())
318
+
319
+ def signal_handler() -> None:
320
+ # prevents the uvicorn signal handler to exit early
321
+ server_task.cancel()
322
+
323
+ loop.add_signal_handler(signal.SIGINT, signal_handler)
324
+ loop.add_signal_handler(signal.SIGTERM, signal_handler)
325
+
326
+ try:
327
+ await server_task
328
+ except asyncio.CancelledError:
329
+ print("Gracefully stopping http server")
330
+ await server.shutdown()
331
332
333
if __name__ == "__main__":
@@ -314,4 +337,4 @@ if __name__ == "__main__":
337
description="vLLM OpenAI-Compatible RESTful API server.")
338
parser = make_arg_parser(parser)
339
args = parser.parse_args()
317
- run_server(args)
340
+ asyncio.run(run_server(args))
vllm/scripts.py
+2
-1
@@ -1,5 +1,6 @@
1
# The CLI entrypoint to vLLM.
2
import argparse
3
+import asyncio
4
import os
5
import signal
6
import sys
@@ -25,7 +26,7 @@ def serve(args: argparse.Namespace) -> None:
26
# EngineArgs expects the model name to be passed as --model.
27
args.model = args.model_tag
28
28
- run_server(args)
29
+ asyncio.run(run_server(args))
30
31
32
def interactive_cli(args: argparse.Namespace) -> None: