@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 890ca36072
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

Revert "[Bugfix] Use host argument to bind to interface (#9798)" (#9852)

Kevin H. Luu committed Oct 30, 2024 at 15:44 UTC 890ca3607208a10514e65cfdf182bdd4125baef6
2 files changed +2 -2
vllm/entrypoints/openai/api_server.py
+1 -1
@@ -544,7 +544,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:
544 # This avoids race conditions with ray.
545 # see https://github.com/vllm-project/vllm/issues/8204
546 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
547 - sock.bind((args.host, args.port))
547 + sock.bind(("", args.port))
548
549 def signal_handler(*_) -> None:
550 # Interrupt server on sigterm while initializing
vllm/entrypoints/openai/cli_args.py
+1 -1
@@ -77,7 +77,7 @@ class PromptAdapterParserAction(argparse.Action):
77 def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
78 parser.add_argument("--host",
79 type=nullable_str,
80 - default="0.0.0.0",
80 + default=None,
81 help="host name")
82 parser.add_argument("--port", type=int, default=8000, help="port number")
83 parser.add_argument(