@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 08133c4d1a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add SSL arguments to API servers (#2109)

Harry Mellor committed Dec 18, 2023 at 02:56 UTC 08133c4d1a9e3c13aa42c3fb83aa7c987e79289f
2 files changed +16 -2
vllm/entrypoints/api_server.py
+5 -1
@@ -73,6 +73,8 @@ if __name__ == "__main__":
73 parser = argparse.ArgumentParser()
74 parser.add_argument("--host", type=str, default=None)
75 parser.add_argument("--port", type=int, default=8000)
76 + parser.add_argument("--ssl-keyfile", type=str, default=None)
77 + parser.add_argument("--ssl-certfile", type=str, default=None)
78 parser = AsyncEngineArgs.add_cli_args(parser)
79 args = parser.parse_args()
80
@@ -83,4 +85,6 @@ if __name__ == "__main__":
85 host=args.host,
86 port=args.port,
87 log_level="debug",
86 - timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
88 + timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
89 + ssl_keyfile=args.ssl_keyfile,
90 + ssl_certfile=args.ssl_certfile)
vllm/entrypoints/openai/api_server.py
+11 -1
@@ -80,6 +80,14 @@ def parse_args():
80 default="assistant",
81 help="The role name to return if "
82 "`request.add_generation_prompt=true`.")
83 + parser.add_argument("--ssl-keyfile",
84 + type=str,
85 + default=None,
86 + help="The file path to the SSL key file")
87 + parser.add_argument("--ssl-certfile",
88 + type=str,
89 + default=None,
90 + help="The file path to the SSL cert file")
91
92 parser = AsyncEngineArgs.add_cli_args(parser)
93 return parser.parse_args()
@@ -744,4 +752,6 @@ if __name__ == "__main__":
752 host=args.host,
753 port=args.port,
754 log_level="info",
747 - timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
755 + timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
756 + ssl_keyfile=args.ssl_keyfile,
757 + ssl_certfile=args.ssl_certfile)