A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Ensure metrics are logged regardless of requests (#2347)
Iskren Ivov Chernev committed
Jan 5, 2024 at 15:24 UTC
d0215a58e78572d91dadafe9d832a2db89b09a13
3 files changed
+28
-1
vllm/engine/async_llm_engine.py
+6
@@ -506,3 +506,9 @@ class AsyncLLMEngine:
506
max_log_len=engine_args.max_log_len,
507
start_engine_loop=start_engine_loop)
508
return engine
509
+
510
+ async def do_log_stats(self) -> None:
511
+ if self.engine_use_ray:
512
+ await self.engine.do_log_stats.remote()
513
+ else:
514
+ self.engine.do_log_stats()
vllm/engine/llm_engine.py
+3
@@ -641,6 +641,9 @@ class LLMEngine:
641
642
return self._process_model_outputs(output, scheduler_outputs)
643
644
+ def do_log_stats(self) -> None:
645
+ self._log_system_stats(False, 0)
646
+
647
def _log_system_stats(
648
self,
649
prompt_run: bool,
vllm/entrypoints/openai/api_server.py
+19
-1
@@ -6,6 +6,7 @@ import asyncio
6
import codecs
7
import json
8
import time
9
+from contextlib import asynccontextmanager
10
from http import HTTPStatus
11
from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union
12
@@ -38,11 +39,28 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
39
40
logger = init_logger(__name__)
41
served_model = None
41
-app = fastapi.FastAPI()
42
+engine_args = None
43
engine = None
44
response_role = None
45
46
47
+@asynccontextmanager
48
+async def lifespan(app: fastapi.FastAPI):
49
+
50
+ async def _force_log():
51
+ while True:
52
+ await asyncio.sleep(10)
53
+ await engine.do_log_stats()
54
+
55
+ if not engine_args.disable_log_stats:
56
+ asyncio.create_task(_force_log())
57
+
58
+ yield
59
+
60
+
61
+app = fastapi.FastAPI(lifespan=lifespan)
62
+
63
+
64
def parse_args():
65
parser = argparse.ArgumentParser(
66
description="vLLM OpenAI-Compatible RESTful API server.")