A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add SSL arguments to API servers (#2109)
Harry Mellor committed
Dec 18, 2023 at 02:56 UTC
08133c4d1a9e3c13aa42c3fb83aa7c987e79289f
2 files changed
+16
-2
vllm/entrypoints/api_server.py
+5
-1
@@ -73,6 +73,8 @@ if __name__ == "__main__":
73
parser = argparse.ArgumentParser()
74
parser.add_argument("--host", type=str, default=None)
75
parser.add_argument("--port", type=int, default=8000)
76
+ parser.add_argument("--ssl-keyfile", type=str, default=None)
77
+ parser.add_argument("--ssl-certfile", type=str, default=None)
78
parser = AsyncEngineArgs.add_cli_args(parser)
79
args = parser.parse_args()
80
@@ -83,4 +85,6 @@ if __name__ == "__main__":
85
host=args.host,
86
port=args.port,
87
log_level="debug",
86
- timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
88
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
89
+ ssl_keyfile=args.ssl_keyfile,
90
+ ssl_certfile=args.ssl_certfile)
vllm/entrypoints/openai/api_server.py
+11
-1
@@ -80,6 +80,14 @@ def parse_args():
80
default="assistant",
81
help="The role name to return if "
82
"`request.add_generation_prompt=true`.")
83
+ parser.add_argument("--ssl-keyfile",
84
+ type=str,
85
+ default=None,
86
+ help="The file path to the SSL key file")
87
+ parser.add_argument("--ssl-certfile",
88
+ type=str,
89
+ default=None,
90
+ help="The file path to the SSL cert file")
91
92
parser = AsyncEngineArgs.add_cli_args(parser)
93
return parser.parse_args()
@@ -744,4 +752,6 @@ if __name__ == "__main__":
752
host=args.host,
753
port=args.port,
754
log_level="info",
747
- timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
755
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
756
+ ssl_keyfile=args.ssl_keyfile,
757
+ ssl_certfile=args.ssl_certfile)