@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 554af9228d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] use AF_INET6 for OpenAI Compatible Server with ipv6 (#9583)

Signed-off-by: xiaozijin <xiaozijin@bytedance.com>

Zijin Xiao committed Nov 15, 2024 at 08:38 UTC 554af9228df620a63d4736240a8f76a64a675f4d
1 file changed +17 -5
vllm/entrypoints/openai/api_server.py
+17 -5
@@ -12,7 +12,7 @@ from argparse import Namespace
12 from contextlib import asynccontextmanager
13 from functools import partial
14 from http import HTTPStatus
15 -from typing import AsyncIterator, Optional, Set
15 +from typing import AsyncIterator, Optional, Set, Tuple
16
17 import uvloop
18 from fastapi import APIRouter, FastAPI, Request
@@ -57,7 +57,8 @@ from vllm.entrypoints.openai.serving_tokenization import (
57 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
58 from vllm.logger import init_logger
59 from vllm.usage.usage_lib import UsageContext
60 -from vllm.utils import FlexibleArgumentParser, get_open_zmq_ipc_path
60 +from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
61 + is_valid_ipv6_address)
62 from vllm.version import __version__ as VLLM_VERSION
63
64 if envs.VLLM_USE_V1:
@@ -568,6 +569,18 @@ def init_app_state(
569 )
570
571
572 +def create_server_socket(addr: Tuple[str, int]) -> socket.socket:
573 + family = socket.AF_INET
574 + if is_valid_ipv6_address(addr[0]):
575 + family = socket.AF_INET6
576 +
577 + sock = socket.socket(family=family, type=socket.SOCK_STREAM)
578 + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
579 + sock.bind(addr)
580 +
581 + return sock
582 +
583 +
584 async def run_server(args, **uvicorn_kwargs) -> None:
585 logger.info("vLLM API server version %s", VLLM_VERSION)
586 logger.info("args: %s", args)
@@ -584,9 +597,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:
597 # workaround to make sure that we bind the port before the engine is set up.
598 # This avoids race conditions with ray.
599 # see https://github.com/vllm-project/vllm/issues/8204
587 - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
588 - sock.bind((args.host or "", args.port))
589 - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
600 + sock_addr = (args.host or "", args.port)
601 + sock = create_server_socket(sock_addr)
602
603 def signal_handler(*_) -> None:
604 # Interrupt server on sigterm while initializing