@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 808632d3b4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[BugFix] Prevent the task of `_force_log` from being garbage collected (#4567)

Yang, Bo committed May 2, 2024 at 18:35 UTC 808632d3b4effd3c0807325b529d0354894c31b1
1 file changed +6 -1
vllm/entrypoints/openai/api_server.py
+6 -1
@@ -4,6 +4,7 @@ import inspect
4 import re
5 from contextlib import asynccontextmanager
6 from http import HTTPStatus
7 +from typing import Any, Set
8
9 import fastapi
10 import uvicorn
@@ -33,6 +34,8 @@ openai_serving_chat: OpenAIServingChat
34 openai_serving_completion: OpenAIServingCompletion
35 logger = init_logger(__name__)
36
37 +_running_tasks: Set[asyncio.Task[Any]] = set()
38 +
39
40 @asynccontextmanager
41 async def lifespan(app: fastapi.FastAPI):
@@ -43,7 +46,9 @@ async def lifespan(app: fastapi.FastAPI):
46 await engine.do_log_stats()
47
48 if not engine_args.disable_log_stats:
46 - asyncio.create_task(_force_log())
49 + task = asyncio.create_task(_force_log())
50 + _running_tasks.add(task)
51 + task.add_done_callback(_running_tasks.remove)
52
53 yield
54