@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ee812580f7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Frontend] split run_server into build_server and run_server (#6740)

Daniele committed Jul 24, 2024 at 19:36 UTC ee812580f7ae3969cf2550cc4dab31490bfea950
2 files changed +52 -28
vllm/entrypoints/openai/api_server.py
+50 -27
@@ -2,6 +2,7 @@ import asyncio
2 import importlib
3 import inspect
4 import re
5 +import signal
6 from contextlib import asynccontextmanager
7 from http import HTTPStatus
8 from typing import Optional, Set
@@ -213,12 +214,13 @@ def build_app(args):
214 return app
215
216
216 -def run_server(args, llm_engine=None):
217 +async def build_server(
218 + args,
219 + llm_engine: Optional[AsyncLLMEngine] = None,
220 + **uvicorn_kwargs,
221 +) -> uvicorn.Server:
222 app = build_app(args)
223
219 - logger.info("vLLM API server version %s", VLLM_VERSION)
220 - logger.info("args: %s", args)
221 -
224 if args.served_model_name is not None:
225 served_model_names = args.served_model_name
226 else:
@@ -231,19 +233,7 @@ def run_server(args, llm_engine=None):
233 if llm_engine is not None else AsyncLLMEngine.from_engine_args(
234 engine_args, usage_context=UsageContext.OPENAI_API_SERVER))
235
234 - event_loop: Optional[asyncio.AbstractEventLoop]
235 - try:
236 - event_loop = asyncio.get_running_loop()
237 - except RuntimeError:
238 - event_loop = None
239 -
240 - if event_loop is not None and event_loop.is_running():
241 - # If the current is instanced by Ray Serve,
242 - # there is already a running event loop
243 - model_config = event_loop.run_until_complete(engine.get_model_config())
244 - else:
245 - # When using single vLLM without engine_use_ray
246 - model_config = asyncio.run(engine.get_model_config())
236 + model_config = await engine.get_model_config()
237
238 if args.disable_log_requests:
239 request_logger = None
@@ -296,15 +286,48 @@ def run_server(args, llm_engine=None):
286 methods = ', '.join(route.methods)
287 logger.info("Route: %s, Methods: %s", route.path, methods)
288
299 - uvicorn.run(app,
300 - host=args.host,
301 - port=args.port,
302 - log_level=args.uvicorn_log_level,
303 - timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
304 - ssl_keyfile=args.ssl_keyfile,
305 - ssl_certfile=args.ssl_certfile,
306 - ssl_ca_certs=args.ssl_ca_certs,
307 - ssl_cert_reqs=args.ssl_cert_reqs)
289 + config = uvicorn.Config(
290 + app,
291 + host=args.host,
292 + port=args.port,
293 + log_level=args.uvicorn_log_level,
294 + timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
295 + ssl_keyfile=args.ssl_keyfile,
296 + ssl_certfile=args.ssl_certfile,
297 + ssl_ca_certs=args.ssl_ca_certs,
298 + ssl_cert_reqs=args.ssl_cert_reqs,
299 + **uvicorn_kwargs,
300 + )
301 +
302 + return uvicorn.Server(config)
303 +
304 +
305 +async def run_server(args, llm_engine=None, **uvicorn_kwargs) -> None:
306 + logger.info("vLLM API server version %s", VLLM_VERSION)
307 + logger.info("args: %s", args)
308 +
309 + server = await build_server(
310 + args,
311 + llm_engine,
312 + **uvicorn_kwargs,
313 + )
314 +
315 + loop = asyncio.get_running_loop()
316 +
317 + server_task = loop.create_task(server.serve())
318 +
319 + def signal_handler() -> None:
320 + # prevents the uvicorn signal handler to exit early
321 + server_task.cancel()
322 +
323 + loop.add_signal_handler(signal.SIGINT, signal_handler)
324 + loop.add_signal_handler(signal.SIGTERM, signal_handler)
325 +
326 + try:
327 + await server_task
328 + except asyncio.CancelledError:
329 + print("Gracefully stopping http server")
330 + await server.shutdown()
331
332
333 if __name__ == "__main__":
@@ -314,4 +337,4 @@ if __name__ == "__main__":
337 description="vLLM OpenAI-Compatible RESTful API server.")
338 parser = make_arg_parser(parser)
339 args = parser.parse_args()
317 - run_server(args)
340 + asyncio.run(run_server(args))
vllm/scripts.py
+2 -1
@@ -1,5 +1,6 @@
1 # The CLI entrypoint to vLLM.
2 import argparse
3 +import asyncio
4 import os
5 import signal
6 import sys
@@ -25,7 +26,7 @@ def serve(args: argparse.Namespace) -> None:
26 # EngineArgs expects the model name to be passed as --model.
27 args.model = args.model_tag
28
28 - run_server(args)
29 + asyncio.run(run_server(args))
30
31
32 def interactive_cli(args: argparse.Namespace) -> None: