A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Frontend] enable custom logging for the uvicorn server (OpenAI API server) (#18403)
Signed-off-by: François Paupier <francois.paupier@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Frαnçois committed
Jun 2, 2025 at 17:04 UTC
20133cfee2e8a21d18be08f90b611cf0424cce96
2 files changed
+27
-2
vllm/entrypoints/openai/api_server.py
+19
-2
@@ -5,6 +5,7 @@ import atexit
5
import gc
6
import importlib
7
import inspect
8
+import json
9
import multiprocessing
10
import os
11
import signal
@@ -16,7 +17,6 @@ from collections.abc import AsyncIterator
17
from contextlib import asynccontextmanager
18
from functools import partial
19
from http import HTTPStatus
19
-from json import JSONDecodeError
20
from typing import Annotated, Any, Optional
21
22
import prometheus_client
@@ -930,7 +930,7 @@ async def invocations(raw_request: Request):
930
"""
931
try:
932
body = await raw_request.json()
933
- except JSONDecodeError as e:
933
+ except json.JSONDecodeError as e:
934
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST.value,
935
detail=f"JSON decode error: {e}") from e
936
@@ -1003,6 +1003,18 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
1003
return Response(status_code=200, content=response)
1004
1005
1006
+def load_log_config(log_config_file: Optional[str]) -> Optional[dict]:
1007
+ if not log_config_file:
1008
+ return None
1009
+ try:
1010
+ with open(log_config_file) as f:
1011
+ return json.load(f)
1012
+ except Exception as e:
1013
+ logger.warning("Failed to load log config from file %s: error %s",
1014
+ log_config_file, e)
1015
+ return None
1016
+
1017
+
1018
def build_app(args: Namespace) -> FastAPI:
1019
if args.disable_fastapi_docs:
1020
app = FastAPI(openapi_url=None,
@@ -1324,6 +1336,11 @@ async def run_server_worker(listen_address,
1336
1337
server_index = client_config.get("client_index", 0) if client_config else 0
1338
1339
+ # Load logging config for uvicorn if specified
1340
+ log_config = load_log_config(args.log_config_file)
1341
+ if log_config is not None:
1342
+ uvicorn_kwargs['log_config'] = log_config
1343
+
1344
async with build_async_engine_client(args, client_config) as engine_client:
1345
app = build_app(args)
1346
vllm/entrypoints/openai/cli_args.py
+8
@@ -11,6 +11,7 @@ import ssl
11
from collections.abc import Sequence
12
from typing import Optional, Union, get_args
13
14
+import vllm.envs as envs
15
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
16
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
17
validate_chat_template)
@@ -243,6 +244,13 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
244
" into OpenAI API format, the name register in this plugin can be used "
245
"in ``--tool-call-parser``.")
246
247
+ parser.add_argument(
248
+ "--log-config-file",
249
+ type=str,
250
+ default=envs.VLLM_LOGGING_CONFIG_PATH,
251
+ help="Path to logging config JSON file for both vllm and uvicorn",
252
+ )
253
+
254
parser = AsyncEngineArgs.add_cli_args(parser)
255
256
parser.add_argument('--max-log-len',