@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 03d37f2441
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Fix] Add args for mTLS support (#3430)

Co-authored-by: declark1 <daniel.clark@ibm.com>

Dan Clark committed Mar 15, 2024 at 09:56 UTC 03d37f24413b13a4e42ee115f89f647c441d1fcd
2 files changed +21 -5
vllm/entrypoints/api_server.py
+7 -4
@@ -8,6 +8,7 @@ change `vllm/entrypoints/openai/api_server.py` instead.
8
9 import argparse
10 import json
11 +import ssl
12 from typing import AsyncGenerator
13
14 from fastapi import FastAPI, Request
@@ -86,10 +87,12 @@ if __name__ == "__main__":
87 type=str,
88 default=None,
89 help="The CA certificates file")
89 - parser.add_argument("--ssl-cert-reqs",
90 - type=int,
91 - default=0,
92 - help="Whether client certificate is required")
90 + parser.add_argument(
91 + "--ssl-cert-reqs",
92 + type=int,
93 + default=int(ssl.CERT_NONE),
94 + help="Whether client certificate is required (see stdlib ssl module's)"
95 + )
96 parser.add_argument(
97 "--root-path",
98 type=str,
vllm/entrypoints/openai/api_server.py
+14 -1
@@ -5,6 +5,7 @@ from contextlib import asynccontextmanager
5 import os
6 import importlib
7 import inspect
8 +import ssl
9
10 from prometheus_client import make_asgi_app
11 import fastapi
@@ -124,6 +125,16 @@ def parse_args():
125 type=str,
126 default=None,
127 help="The file path to the SSL cert file")
128 + parser.add_argument("--ssl-ca-certs",
129 + type=str,
130 + default=None,
131 + help="The CA certificates file")
132 + parser.add_argument(
133 + "--ssl-cert-reqs",
134 + type=int,
135 + default=int(ssl.CERT_NONE),
136 + help="Whether client certificate is required (see stdlib ssl module's)"
137 + )
138 parser.add_argument(
139 "--root-path",
140 type=str,
@@ -262,4 +273,6 @@ if __name__ == "__main__":
273 log_level=args.uvicorn_log_level,
274 timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
275 ssl_keyfile=args.ssl_keyfile,
265 - ssl_certfile=args.ssl_certfile)
276 + ssl_certfile=args.ssl_certfile,
277 + ssl_ca_certs=args.ssl_ca_certs,
278 + ssl_cert_reqs=args.ssl_cert_reqs)