A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Fix] Add args for mTLS support (#3430)
Co-authored-by: declark1 <daniel.clark@ibm.com>
Dan Clark committed
Mar 15, 2024 at 09:56 UTC
03d37f24413b13a4e42ee115f89f647c441d1fcd
2 files changed
+21
-5
vllm/entrypoints/api_server.py
+7
-4
@@ -8,6 +8,7 @@ change `vllm/entrypoints/openai/api_server.py` instead.
8
9
import argparse
10
import json
11
+import ssl
12
from typing import AsyncGenerator
13
14
from fastapi import FastAPI, Request
@@ -86,10 +87,12 @@ if __name__ == "__main__":
87
type=str,
88
default=None,
89
help="The CA certificates file")
89
- parser.add_argument("--ssl-cert-reqs",
90
- type=int,
91
- default=0,
92
- help="Whether client certificate is required")
90
+ parser.add_argument(
91
+ "--ssl-cert-reqs",
92
+ type=int,
93
+ default=int(ssl.CERT_NONE),
94
+ help="Whether client certificate is required (see stdlib ssl module's)"
95
+ )
96
parser.add_argument(
97
"--root-path",
98
type=str,
vllm/entrypoints/openai/api_server.py
+14
-1
@@ -5,6 +5,7 @@ from contextlib import asynccontextmanager
5
import os
6
import importlib
7
import inspect
8
+import ssl
9
10
from prometheus_client import make_asgi_app
11
import fastapi
@@ -124,6 +125,16 @@ def parse_args():
125
type=str,
126
default=None,
127
help="The file path to the SSL cert file")
128
+ parser.add_argument("--ssl-ca-certs",
129
+ type=str,
130
+ default=None,
131
+ help="The CA certificates file")
132
+ parser.add_argument(
133
+ "--ssl-cert-reqs",
134
+ type=int,
135
+ default=int(ssl.CERT_NONE),
136
+ help="Whether client certificate is required (see stdlib ssl module's)"
137
+ )
138
parser.add_argument(
139
"--root-path",
140
type=str,
@@ -262,4 +273,6 @@ if __name__ == "__main__":
273
log_level=args.uvicorn_log_level,
274
timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
275
ssl_keyfile=args.ssl_keyfile,
265
- ssl_certfile=args.ssl_certfile)
276
+ ssl_certfile=args.ssl_certfile,
277
+ ssl_ca_certs=args.ssl_ca_certs,
278
+ ssl_cert_reqs=args.ssl_cert_reqs)