@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 578087e56c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Frontend] Pass pre-created socket to uvicorn (#13113)

Russell Bryant committed Feb 13, 2025 at 03:51 UTC 578087e56c21b2c940135a7c059c8dd88c8961f5
3 files changed +17 -6
vllm/entrypoints/api_server.py
+1
@@ -127,6 +127,7 @@ async def run_server(args: Namespace,
127
128 shutdown_task = await serve_http(
129 app,
130 + sock=None,
131 host=args.host,
132 port=args.port,
133 log_level=args.log_level,
vllm/entrypoints/launcher.py
+6 -3
@@ -2,8 +2,9 @@
2
3 import asyncio
4 import signal
5 +import socket
6 from http import HTTPStatus
6 -from typing import Any
7 +from typing import Any, Optional
8
9 import uvicorn
10 from fastapi import FastAPI, Request, Response
@@ -17,7 +18,8 @@ from vllm.utils import find_process_using_port
18 logger = init_logger(__name__)
19
20
20 -async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
21 +async def serve_http(app: FastAPI, sock: Optional[socket.socket],
22 + **uvicorn_kwargs: Any):
23 logger.info("Available routes are:")
24 for route in app.routes:
25 methods = getattr(route, "methods", None)
@@ -34,7 +36,8 @@ async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
36
37 loop = asyncio.get_running_loop()
38
37 - server_task = loop.create_task(server.serve())
39 + server_task = loop.create_task(
40 + server.serve(sockets=[sock] if sock else None))
41
42 def signal_handler() -> None:
43 # prevents the uvicorn signal handler to exit early
vllm/entrypoints/openai/api_server.py
+10 -3
@@ -10,7 +10,6 @@ import os
10 import re
11 import signal
12 import socket
13 -import sys
13 import tempfile
14 import uuid
15 from argparse import Namespace
@@ -831,6 +830,7 @@ def create_server_socket(addr: Tuple[str, int]) -> socket.socket:
830
831 sock = socket.socket(family=family, type=socket.SOCK_STREAM)
832 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
833 + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
834 sock.bind(addr)
835
836 return sock
@@ -878,8 +878,17 @@ async def run_server(args, **uvicorn_kwargs) -> None:
878 model_config = await engine_client.get_model_config()
879 await init_app_state(engine_client, model_config, app.state, args)
880
881 + def _listen_addr(a: str) -> str:
882 + if is_valid_ipv6_address(a):
883 + return '[' + a + ']'
884 + return a or "0.0.0.0"
885 +
886 + logger.info("Starting vLLM API server on http://%s:%d",
887 + _listen_addr(sock_addr[0]), sock_addr[1])
888 +
889 shutdown_task = await serve_http(
890 app,
891 + sock=sock,
892 host=args.host,
893 port=args.port,
894 log_level=args.uvicorn_log_level,
@@ -888,8 +897,6 @@ async def run_server(args, **uvicorn_kwargs) -> None:
897 ssl_certfile=args.ssl_certfile,
898 ssl_ca_certs=args.ssl_ca_certs,
899 ssl_cert_reqs=args.ssl_cert_reqs,
891 - # Workaround to work on macOS
892 - fd=sock.fileno() if sys.platform.startswith("darwin") else None,
900 **uvicorn_kwargs,
901 )
902