@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 09ff7f106a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

API server support ipv4 / ipv6 dualstack (#1288)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

Yunfeng Bai committed Oct 7, 2023 at 15:15 UTC 09ff7f106a1e5ad77f6c2941382d5ba4fd5a0879
3 files changed +3 -6
examples/gradio_webserver.py
+1 -1
@@ -39,7 +39,7 @@ def build_demo():
39
40 if __name__ == "__main__":
41 parser = argparse.ArgumentParser()
42 - parser.add_argument("--host", type=str, default="localhost")
42 + parser.add_argument("--host", type=str, default=None)
43 parser.add_argument("--port", type=int, default=8001)
44 parser.add_argument("--model-url",
45 type=str,
vllm/entrypoints/api_server.py
+1 -1
@@ -65,7 +65,7 @@ async def generate(request: Request) -> Response:
65
66 if __name__ == "__main__":
67 parser = argparse.ArgumentParser()
68 - parser.add_argument("--host", type=str, default="localhost")
68 + parser.add_argument("--host", type=str, default=None)
69 parser.add_argument("--port", type=int, default=8000)
70 parser = AsyncEngineArgs.add_cli_args(parser)
71 args = parser.parse_args()
vllm/entrypoints/openai/api_server.py
+1 -4
@@ -567,10 +567,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
567 if __name__ == "__main__":
568 parser = argparse.ArgumentParser(
569 description="vLLM OpenAI-Compatible RESTful API server.")
570 - parser.add_argument("--host",
571 - type=str,
572 - default="localhost",
573 - help="host name")
570 + parser.add_argument("--host", type=str, default=None, help="host name")
571 parser.add_argument("--port", type=int, default=8000, help="port number")
572 parser.add_argument("--allow-credentials",
573 action="store_true",