A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Feature] Simple API token authentication and pluggable middlewares (#1106)
Erfan Al-Hossami committed
Jan 23, 2024 at 18:13 UTC
9c1352eb5736d9e71d37959db44b6a641e898772
2 files changed
+48
-29
docs/source/getting_started/quickstart.rst
+3
-29
@@ -63,38 +63,10 @@ Call ``llm.generate`` to generate the outputs. It adds the input prompts to vLLM
63
64
The code example can also be found in `examples/offline_inference.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference.py>`_.
65
66
-
67
-API Server
68
-----------
69
-
70
-vLLM can be deployed as an LLM service. We provide an example `FastAPI <https://fastapi.tiangolo.com/>`_ server. Check `vllm/entrypoints/api_server.py <https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/api_server.py>`_ for the server implementation. The server uses ``AsyncLLMEngine`` class to support asynchronous processing of incoming requests.
71
-
72
-Start the server:
73
-
74
-.. code-block:: console
75
-
76
- $ python -m vllm.entrypoints.api_server
77
-
78
-By default, this command starts the server at ``http://localhost:8000`` with the OPT-125M model.
79
-
80
-Query the model in shell:
81
-
82
-.. code-block:: console
83
-
84
- $ curl http://localhost:8000/generate \
85
- $ -d '{
86
- $ "prompt": "San Francisco is a",
87
- $ "use_beam_search": true,
88
- $ "n": 4,
89
- $ "temperature": 0
90
- $ }'
91
-
92
-See `examples/api_client.py <https://github.com/vllm-project/vllm/blob/main/examples/api_client.py>`_ for a more detailed client example.
93
-
66
OpenAI-Compatible Server
67
------------------------
68
97
-vLLM can be deployed as a server that mimics the OpenAI API protocol. This allows vLLM to be used as a drop-in replacement for applications using OpenAI API.
69
+vLLM can be deployed as a server that implements the OpenAI API protocol. This allows vLLM to be used as a drop-in replacement for applications using OpenAI API.
70
By default, it starts the server at ``http://localhost:8000``. You can specify the address with ``--host`` and ``--port`` arguments. The server currently hosts one model at a time (OPT-125M in the command below) and implements `list models <https://platform.openai.com/docs/api-reference/models/list>`_, `create chat completion <https://platform.openai.com/docs/api-reference/chat/completions/create>`_, and `create completion <https://platform.openai.com/docs/api-reference/completions/create>`_ endpoints. We are actively adding support for more endpoints.
71
72
Start the server:
@@ -118,6 +90,8 @@ This server can be queried in the same format as OpenAI API. For example, list t
90
91
$ curl http://localhost:8000/v1/models
92
93
+You can pass in the argument ``--api-key`` or environment variable ``VLLM_API_KEY`` to enable the server to check for API key in the header.
94
+
95
Using OpenAI Completions API with vLLM
96
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
97
vllm/entrypoints/openai/api_server.py
+45
@@ -2,6 +2,10 @@ import argparse
2
import asyncio
3
import json
4
from contextlib import asynccontextmanager
5
+import os
6
+import importlib
7
+import inspect
8
+
9
from aioprometheus import MetricsMiddleware
10
from aioprometheus.asgi.starlette import metrics
11
import fastapi
@@ -64,6 +68,13 @@ def parse_args():
68
type=json.loads,
69
default=["*"],
70
help="allowed headers")
71
+ parser.add_argument(
72
+ "--api-key",
73
+ type=str,
74
+ default=None,
75
+ help=
76
+ "If provided, the server will require this key to be presented in the header."
77
+ )
78
parser.add_argument("--served-model-name",
79
type=str,
80
default=None,
@@ -94,6 +105,17 @@ def parse_args():
105
type=str,
106
default=None,
107
help="FastAPI root_path when app is behind a path based routing proxy")
108
+ parser.add_argument(
109
+ "--middleware",
110
+ type=str,
111
+ action="append",
112
+ default=[],
113
+ help="Additional ASGI middleware to apply to the app. "
114
+ "We accept multiple --middleware arguments. "
115
+ "The value should be an import path. "
116
+ "If a function is provided, vLLM will add it to the server using @app.middleware('http'). "
117
+ "If a class is provided, vLLM will add it to the server using app.add_middleware(). "
118
+ )
119
120
parser = AsyncEngineArgs.add_cli_args(parser)
121
return parser.parse_args()
@@ -161,6 +183,29 @@ if __name__ == "__main__":
183
allow_headers=args.allowed_headers,
184
)
185
186
+ if token := os.environ.get("VLLM_API_KEY") or args.api_key:
187
+
188
+ @app.middleware("http")
189
+ async def authentication(request: Request, call_next):
190
+ if not request.url.path.startswith("/v1"):
191
+ return await call_next(request)
192
+ if request.headers.get("Authorization") != "Bearer " + token:
193
+ return JSONResponse(content={"error": "Unauthorized"},
194
+ status_code=401)
195
+ return await call_next(request)
196
+
197
+ for middleware in args.middleware:
198
+ module_path, object_name = middleware.rsplit(".", 1)
199
+ imported = getattr(importlib.import_module(module_path), object_name)
200
+ if inspect.isclass(imported):
201
+ app.add_middleware(imported)
202
+ elif inspect.iscoroutinefunction(imported):
203
+ app.middleware("http")(imported)
204
+ else:
205
+ raise ValueError(
206
+ f"Invalid middleware {middleware}. Must be a function or a class."
207
+ )
208
+
209
logger.info(f"args: {args}")
210
211
if args.served_model_name is not None: