| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import asyncio |
| 4 | import json |
| 5 | from argparse import Namespace |
| 6 | from contextlib import asynccontextmanager |
| 7 | |
| 8 | from fastapi import FastAPI |
| 9 | |
| 10 | from vllm import envs |
| 11 | from vllm.config.logging import LogFormatter |
| 12 | from vllm.engine.protocol import EngineClient |
| 13 | from vllm.logger import init_logger |
| 14 | from vllm.utils.gc_utils import freeze_gc_heap |
| 15 | |
| 16 | logger = init_logger(__name__) |
| 17 | |
| 18 | |
| 19 | def load_log_config(log_config_file: str | None) -> dict | None: |
| 20 | if not log_config_file: |
| 21 | return None |
| 22 | try: |
| 23 | with open(log_config_file) as f: |
| 24 | return json.load(f) |
| 25 | except Exception as e: |
| 26 | logger.warning( |
| 27 | "Failed to load log config from file %s: error %s", log_config_file, e |
| 28 | ) |
| 29 | return None |
| 30 | |
| 31 | |
| 32 | def get_uvicorn_log_config(args: Namespace) -> dict | None: |
| 33 | """Get the uvicorn log config based on the provided arguments. |
| 34 | |
| 35 | Priority: |
| 36 | 1. If log_config_file is specified, use it |
| 37 | 2. If disable_access_log_for_endpoints or JSON formatting is specified, |
| 38 | create a config with the requested filter and format |
| 39 | 3. Otherwise, return None (use uvicorn defaults) |
| 40 | """ |
| 41 | # First, try to load from file if specified |
| 42 | logging_config = getattr(args, "logging_config", None) |
| 43 | log_config_file = ( |
| 44 | logging_config.pylogging_config_file |
| 45 | if logging_config is not None |
| 46 | else getattr(args, "log_config_file", None) |
| 47 | ) |
| 48 | log_config = load_log_config(log_config_file) |
| 49 | if log_config is not None: |
| 50 | return log_config |
| 51 | |
| 52 | formatter: LogFormatter = ( |
| 53 | logging_config.formatter if logging_config is not None else "text" |
| 54 | ) |
| 55 | if args.disable_access_log_for_endpoints or formatter == "json": |
| 56 | from vllm.logging_utils import create_uvicorn_log_config |
| 57 | |
| 58 | excluded_paths = [ |
| 59 | p.strip() |
| 60 | for p in (args.disable_access_log_for_endpoints or "").split(",") |
| 61 | if p.strip() |
| 62 | ] |
| 63 | return create_uvicorn_log_config( |
| 64 | excluded_paths=excluded_paths, |
| 65 | log_level=args.uvicorn_log_level, |
| 66 | formatter=formatter, |
| 67 | ) |
| 68 | |
| 69 | return None |
| 70 | |
| 71 | |
| 72 | _running_tasks: set[asyncio.Task] = set() |
| 73 | |
| 74 | |
| 75 | @asynccontextmanager |
| 76 | async def lifespan(app: FastAPI): |
| 77 | try: |
| 78 | if app.state.log_stats: |
| 79 | engine_client: EngineClient = app.state.engine_client |
| 80 | |
| 81 | async def _force_log(): |
| 82 | while True: |
| 83 | await asyncio.sleep(envs.VLLM_LOG_STATS_INTERVAL) |
| 84 | await engine_client.do_log_stats() |
| 85 | |
| 86 | task = asyncio.create_task(_force_log()) |
| 87 | _running_tasks.add(task) |
| 88 | task.add_done_callback(_running_tasks.remove) |
| 89 | else: |
| 90 | task = None |
| 91 | |
| 92 | # Mark the startup heap as static so that it's ignored by GC. |
| 93 | # Reduces pause times of oldest generation collections. |
| 94 | freeze_gc_heap() |
| 95 | try: |
| 96 | yield |
| 97 | finally: |
| 98 | if task is not None: |
| 99 | task.cancel() |
| 100 | for attr_name in ( |
| 101 | "openai_serving_transcription", |
| 102 | "openai_serving_translation", |
| 103 | ): |
| 104 | serving = getattr(app.state, attr_name, None) |
| 105 | if serving is not None and hasattr(serving, "shutdown"): |
| 106 | serving.shutdown() |
| 107 | finally: |
| 108 | # Ensure app state including engine ref is gc'd |
| 109 | del app.state |