A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Use host argument to bind to interface (#9798)
Sven Seeberg committed
Oct 29, 2024 at 18:37 UTC
0f43387157010bf84da05c68fc5ff366b3252f01
2 files changed
+2
-2
vllm/entrypoints/openai/api_server.py
+1
-1
@@ -541,7 +541,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:
541
# This avoids race conditions with ray.
542
# see https://github.com/vllm-project/vllm/issues/8204
543
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
544
- sock.bind(("", args.port))
544
+ sock.bind((args.host, args.port))
545
546
def signal_handler(*_) -> None:
547
# Interrupt server on sigterm while initializing
vllm/entrypoints/openai/cli_args.py
+1
-1
@@ -77,7 +77,7 @@ class PromptAdapterParserAction(argparse.Action):
77
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
78
parser.add_argument("--host",
79
type=nullable_str,
80
- default=None,
80
+ default="0.0.0.0",
81
help="host name")
82
parser.add_argument("--port", type=int, default=8000, help="port number")
83
parser.add_argument(