A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[1/N] API Server (Remove Proxy) (#11529)
Robert Shaw committed
Dec 26, 2024 at 18:03 UTC
720b10fdc6891b2540fce172b63eb07c1ba48958
2 files changed
+17
-7
vllm/entrypoints/openai/api_server.py
+12
-6
@@ -585,12 +585,18 @@ def build_app(args: Namespace) -> FastAPI:
585
status_code=401)
586
return await call_next(request)
587
588
- @app.middleware("http")
589
- async def add_request_id(request: Request, call_next):
590
- request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex
591
- response = await call_next(request)
592
- response.headers["X-Request-Id"] = request_id
593
- return response
588
+ if args.enable_request_id_headers:
589
+ logger.warning(
590
+ "CAUTION: Enabling X-Request-Id headers in the API Server. "
591
+ "This can harm performance at high QPS.")
592
+
593
+ @app.middleware("http")
594
+ async def add_request_id(request: Request, call_next):
595
+ request_id = request.headers.get(
596
+ "X-Request-Id") or uuid.uuid4().hex
597
+ response = await call_next(request)
598
+ response.headers["X-Request-Id"] = request_id
599
+ return response
600
601
for middleware in args.middleware:
602
module_path, object_name = middleware.rsplit(".", 1)
vllm/entrypoints/openai/cli_args.py
+5
-1
@@ -196,7 +196,11 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
196
action="store_true",
197
help="If specified, will run the OpenAI frontend server in the same "
198
"process as the model serving engine.")
199
-
199
+ parser.add_argument(
200
+ "--enable-request-id-headers",
201
+ action="store_true",
202
+ help="If specified, API server will add X-Request-Id header to "
203
+ "responses. Caution: this hurts performance at high QPS.")
204
parser.add_argument(
205
"--enable-auto-tool-choice",
206
action="store_true",