@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 720b10fdc6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[1/N] API Server (Remove Proxy) (#11529)

Robert Shaw committed Dec 26, 2024 at 18:03 UTC 720b10fdc6891b2540fce172b63eb07c1ba48958
2 files changed +17 -7
vllm/entrypoints/openai/api_server.py
+12 -6
@@ -585,12 +585,18 @@ def build_app(args: Namespace) -> FastAPI:
585 status_code=401)
586 return await call_next(request)
587
588 - @app.middleware("http")
589 - async def add_request_id(request: Request, call_next):
590 - request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex
591 - response = await call_next(request)
592 - response.headers["X-Request-Id"] = request_id
593 - return response
588 + if args.enable_request_id_headers:
589 + logger.warning(
590 + "CAUTION: Enabling X-Request-Id headers in the API Server. "
591 + "This can harm performance at high QPS.")
592 +
593 + @app.middleware("http")
594 + async def add_request_id(request: Request, call_next):
595 + request_id = request.headers.get(
596 + "X-Request-Id") or uuid.uuid4().hex
597 + response = await call_next(request)
598 + response.headers["X-Request-Id"] = request_id
599 + return response
600
601 for middleware in args.middleware:
602 module_path, object_name = middleware.rsplit(".", 1)
vllm/entrypoints/openai/cli_args.py
+5 -1
@@ -196,7 +196,11 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
196 action="store_true",
197 help="If specified, will run the OpenAI frontend server in the same "
198 "process as the model serving engine.")
199 -
199 + parser.add_argument(
200 + "--enable-request-id-headers",
201 + action="store_true",
202 + help="If specified, API server will add X-Request-Id header to "
203 + "responses. Caution: this hurts performance at high QPS.")
204 parser.add_argument(
205 "--enable-auto-tool-choice",
206 action="store_true",