A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[BugFix] Prevent the task of `_force_log` from being garbage collected (#4567)
Yang, Bo committed
May 2, 2024 at 18:35 UTC
808632d3b4effd3c0807325b529d0354894c31b1
1 file changed
+6
-1
vllm/entrypoints/openai/api_server.py
+6
-1
@@ -4,6 +4,7 @@ import inspect
4
import re
5
from contextlib import asynccontextmanager
6
from http import HTTPStatus
7
+from typing import Any, Set
8
9
import fastapi
10
import uvicorn
@@ -33,6 +34,8 @@ openai_serving_chat: OpenAIServingChat
34
openai_serving_completion: OpenAIServingCompletion
35
logger = init_logger(__name__)
36
37
+_running_tasks: Set[asyncio.Task[Any]] = set()
38
+
39
40
@asynccontextmanager
41
async def lifespan(app: fastapi.FastAPI):
@@ -43,7 +46,9 @@ async def lifespan(app: fastapi.FastAPI):
46
await engine.do_log_stats()
47
48
if not engine_args.disable_log_stats:
46
- asyncio.create_task(_force_log())
49
+ task = asyncio.create_task(_force_log())
50
+ _running_tasks.add(task)
51
+ task.add_done_callback(_running_tasks.remove)
52
53
yield
54