@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d0215a58e7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Ensure metrics are logged regardless of requests (#2347)

Iskren Ivov Chernev committed Jan 5, 2024 at 15:24 UTC d0215a58e78572d91dadafe9d832a2db89b09a13
3 files changed +28 -1
vllm/engine/async_llm_engine.py
+6
@@ -506,3 +506,9 @@ class AsyncLLMEngine:
506 max_log_len=engine_args.max_log_len,
507 start_engine_loop=start_engine_loop)
508 return engine
509 +
510 + async def do_log_stats(self) -> None:
511 + if self.engine_use_ray:
512 + await self.engine.do_log_stats.remote()
513 + else:
514 + self.engine.do_log_stats()
vllm/engine/llm_engine.py
+3
@@ -641,6 +641,9 @@ class LLMEngine:
641
642 return self._process_model_outputs(output, scheduler_outputs)
643
644 + def do_log_stats(self) -> None:
645 + self._log_system_stats(False, 0)
646 +
647 def _log_system_stats(
648 self,
649 prompt_run: bool,
vllm/entrypoints/openai/api_server.py
+19 -1
@@ -6,6 +6,7 @@ import asyncio
6 import codecs
7 import json
8 import time
9 +from contextlib import asynccontextmanager
10 from http import HTTPStatus
11 from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union
12
@@ -38,11 +39,28 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
39
40 logger = init_logger(__name__)
41 served_model = None
41 -app = fastapi.FastAPI()
42 +engine_args = None
43 engine = None
44 response_role = None
45
46
47 +@asynccontextmanager
48 +async def lifespan(app: fastapi.FastAPI):
49 +
50 + async def _force_log():
51 + while True:
52 + await asyncio.sleep(10)
53 + await engine.do_log_stats()
54 +
55 + if not engine_args.disable_log_stats:
56 + asyncio.create_task(_force_log())
57 +
58 + yield
59 +
60 +
61 +app = fastapi.FastAPI(lifespan=lifespan)
62 +
63 +
64 def parse_args():
65 parser = argparse.ArgumentParser(
66 description="vLLM OpenAI-Compatible RESTful API server.")