A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] use AF_INET6 for OpenAI Compatible Server with ipv6 (#9583)
Signed-off-by: xiaozijin <xiaozijin@bytedance.com>
Zijin Xiao committed
Nov 15, 2024 at 08:38 UTC
554af9228df620a63d4736240a8f76a64a675f4d
1 file changed
+17
-5
vllm/entrypoints/openai/api_server.py
+17
-5
@@ -12,7 +12,7 @@ from argparse import Namespace
12
from contextlib import asynccontextmanager
13
from functools import partial
14
from http import HTTPStatus
15
-from typing import AsyncIterator, Optional, Set
15
+from typing import AsyncIterator, Optional, Set, Tuple
16
17
import uvloop
18
from fastapi import APIRouter, FastAPI, Request
@@ -57,7 +57,8 @@ from vllm.entrypoints.openai.serving_tokenization import (
57
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
58
from vllm.logger import init_logger
59
from vllm.usage.usage_lib import UsageContext
60
-from vllm.utils import FlexibleArgumentParser, get_open_zmq_ipc_path
60
+from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
61
+ is_valid_ipv6_address)
62
from vllm.version import __version__ as VLLM_VERSION
63
64
if envs.VLLM_USE_V1:
@@ -568,6 +569,18 @@ def init_app_state(
569
)
570
571
572
+def create_server_socket(addr: Tuple[str, int]) -> socket.socket:
573
+ family = socket.AF_INET
574
+ if is_valid_ipv6_address(addr[0]):
575
+ family = socket.AF_INET6
576
+
577
+ sock = socket.socket(family=family, type=socket.SOCK_STREAM)
578
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
579
+ sock.bind(addr)
580
+
581
+ return sock
582
+
583
+
584
async def run_server(args, **uvicorn_kwargs) -> None:
585
logger.info("vLLM API server version %s", VLLM_VERSION)
586
logger.info("args: %s", args)
@@ -584,9 +597,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:
597
# workaround to make sure that we bind the port before the engine is set up.
598
# This avoids race conditions with ray.
599
# see https://github.com/vllm-project/vllm/issues/8204
587
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
588
- sock.bind((args.host or "", args.port))
589
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
600
+ sock_addr = (args.host or "", args.port)
601
+ sock = create_server_socket(sock_addr)
602
603
def signal_handler(*_) -> None:
604
# Interrupt server on sigterm while initializing