A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix][Core] Fix and refactor logging stats (#4336)
Roy committed
May 2, 2024 at 04:08 UTC
3a922c1e7ee6753f41c6cc9d6d47d3b2d0110447
2 files changed
+16
-10
vllm/engine/async_llm_engine.py
+9
-5
@@ -8,6 +8,7 @@ from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
8
from transformers import PreTrainedTokenizer
9
10
from vllm.config import DecodingConfig, ModelConfig
11
+from vllm.core.scheduler import SchedulerOutputs
12
from vllm.engine.arg_utils import AsyncEngineArgs
13
from vllm.engine.llm_engine import LLMEngine
14
from vllm.executor.ray_utils import initialize_ray_cluster, ray
@@ -15,7 +16,7 @@ from vllm.logger import init_logger
16
from vllm.lora.request import LoRARequest
17
from vllm.outputs import RequestOutput
18
from vllm.sampling_params import SamplingParams
18
-from vllm.sequence import MultiModalData
19
+from vllm.sequence import MultiModalData, SamplerOutput
20
from vllm.usage.usage_lib import UsageContext
21
22
logger = init_logger(__name__)
@@ -224,8 +225,7 @@ class _AsyncLLMEngine(LLMEngine):
225
scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
226
227
# Log stats.
227
- if self.log_stats:
228
- self.stat_logger.log(self._get_stats(scheduler_outputs))
228
+ self.do_log_stats(scheduler_outputs, output)
229
230
return request_outputs
231
@@ -707,9 +707,13 @@ class AsyncLLMEngine:
707
else:
708
return self.engine.get_decoding_config()
709
710
- async def do_log_stats(self) -> None:
710
+ async def do_log_stats(
711
+ self,
712
+ scheduler_outputs: Optional[SchedulerOutputs] = None,
713
+ model_output: Optional[List[SamplerOutput]] = None) -> None:
714
if self.engine_use_ray:
712
- await self.engine.do_log_stats.remote() # type: ignore
715
+ await self.engine.do_log_stats.remote( # type: ignore
716
+ scheduler_outputs, model_output)
717
else:
718
self.engine.do_log_stats()
719
vllm/engine/llm_engine.py
+7
-5
@@ -597,16 +597,18 @@ class LLMEngine:
597
scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
598
599
# Log stats.
600
- if self.log_stats:
601
- self.stat_logger.log(
602
- self._get_stats(scheduler_outputs, model_output=output))
600
+ self.do_log_stats(scheduler_outputs, output)
601
602
return request_outputs
603
606
- def do_log_stats(self) -> None:
604
+ def do_log_stats(
605
+ self,
606
+ scheduler_outputs: Optional[SchedulerOutputs] = None,
607
+ model_output: Optional[List[SamplerOutput]] = None) -> None:
608
"""Forced log when no requests active."""
609
if self.log_stats:
609
- self.stat_logger.log(self._get_stats(scheduler_outputs=None))
610
+ self.stat_logger.log(
611
+ self._get_stats(scheduler_outputs, model_output))
612
613
def _get_stats(
614
self,