A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Feature] Add middleware to log API Server responses (#15593)
Signed-off-by: Yuan Tang <terrytangyuan@gmail.com>
Yuan Tang committed
Mar 27, 2025 at 13:49 UTC
66aa4c0bf4973065b45172dc18346016a6087a10
2 files changed
+21
vllm/entrypoints/openai/api_server.py
+16
@@ -24,6 +24,7 @@ from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request
24
from fastapi.exceptions import RequestValidationError
25
from fastapi.middleware.cors import CORSMiddleware
26
from fastapi.responses import JSONResponse, Response, StreamingResponse
27
+from starlette.concurrency import iterate_in_threadpool
28
from starlette.datastructures import State
29
from starlette.routing import Mount
30
from typing_extensions import assert_never
@@ -846,6 +847,21 @@ def build_app(args: Namespace) -> FastAPI:
847
response.headers["X-Request-Id"] = request_id
848
return response
849
850
+ if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE:
851
+ logger.warning("CAUTION: Enabling log response in the API Server. "
852
+ "This can include sensitive information and should be "
853
+ "avoided in production.")
854
+
855
+ @app.middleware("http")
856
+ async def log_response(request: Request, call_next):
857
+ response = await call_next(request)
858
+ response_body = [
859
+ section async for section in response.body_iterator
860
+ ]
861
+ response.body_iterator = iterate_in_threadpool(iter(response_body))
862
+ logger.info("response_body={%s}", response_body[0].decode())
863
+ return response
864
+
865
for middleware in args.middleware:
866
module_path, object_name = middleware.rsplit(".", 1)
867
imported = getattr(importlib.import_module(module_path), object_name)
vllm/envs.py
+5
@@ -270,6 +270,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
270
"VLLM_API_KEY":
271
lambda: os.environ.get("VLLM_API_KEY", None),
272
273
+ # Whether to log responses from API Server for debugging
274
+ "VLLM_DEBUG_LOG_API_SERVER_RESPONSE":
275
+ lambda: os.environ.get("VLLM_DEBUG_LOG_API_SERVER_RESPONSE", "False").
276
+ lower() == "true",
277
+
278
# S3 access information, used for tensorizer to load model from S3
279
"S3_ACCESS_KEY_ID":
280
lambda: os.environ.get("S3_ACCESS_KEY_ID", None),