A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
API server support ipv4 / ipv6 dualstack (#1288)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
Yunfeng Bai committed
Oct 7, 2023 at 15:15 UTC
09ff7f106a1e5ad77f6c2941382d5ba4fd5a0879
3 files changed
+3
-6
examples/gradio_webserver.py
+1
-1
@@ -39,7 +39,7 @@ def build_demo():
39
40
if __name__ == "__main__":
41
parser = argparse.ArgumentParser()
42
- parser.add_argument("--host", type=str, default="localhost")
42
+ parser.add_argument("--host", type=str, default=None)
43
parser.add_argument("--port", type=int, default=8001)
44
parser.add_argument("--model-url",
45
type=str,
vllm/entrypoints/api_server.py
+1
-1
@@ -65,7 +65,7 @@ async def generate(request: Request) -> Response:
65
66
if __name__ == "__main__":
67
parser = argparse.ArgumentParser()
68
- parser.add_argument("--host", type=str, default="localhost")
68
+ parser.add_argument("--host", type=str, default=None)
69
parser.add_argument("--port", type=int, default=8000)
70
parser = AsyncEngineArgs.add_cli_args(parser)
71
args = parser.parse_args()
vllm/entrypoints/openai/api_server.py
+1
-4
@@ -567,10 +567,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
567
if __name__ == "__main__":
568
parser = argparse.ArgumentParser(
569
description="vLLM OpenAI-Compatible RESTful API server.")
570
- parser.add_argument("--host",
571
- type=str,
572
- default="localhost",
573
- help="host name")
570
+ parser.add_argument("--host", type=str, default=None, help="host name")
571
parser.add_argument("--port", type=int, default=8000, help="port number")
572
parser.add_argument("--allow-credentials",
573
action="store_true",