@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3a922c1e7e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix][Core] Fix and refactor logging stats (#4336)

Roy committed May 2, 2024 at 04:08 UTC 3a922c1e7ee6753f41c6cc9d6d47d3b2d0110447
2 files changed +16 -10
vllm/engine/async_llm_engine.py
+9 -5
@@ -8,6 +8,7 @@ from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
8 from transformers import PreTrainedTokenizer
9
10 from vllm.config import DecodingConfig, ModelConfig
11 +from vllm.core.scheduler import SchedulerOutputs
12 from vllm.engine.arg_utils import AsyncEngineArgs
13 from vllm.engine.llm_engine import LLMEngine
14 from vllm.executor.ray_utils import initialize_ray_cluster, ray
@@ -15,7 +16,7 @@ from vllm.logger import init_logger
16 from vllm.lora.request import LoRARequest
17 from vllm.outputs import RequestOutput
18 from vllm.sampling_params import SamplingParams
18 -from vllm.sequence import MultiModalData
19 +from vllm.sequence import MultiModalData, SamplerOutput
20 from vllm.usage.usage_lib import UsageContext
21
22 logger = init_logger(__name__)
@@ -224,8 +225,7 @@ class _AsyncLLMEngine(LLMEngine):
225 scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
226
227 # Log stats.
227 - if self.log_stats:
228 - self.stat_logger.log(self._get_stats(scheduler_outputs))
228 + self.do_log_stats(scheduler_outputs, output)
229
230 return request_outputs
231
@@ -707,9 +707,13 @@ class AsyncLLMEngine:
707 else:
708 return self.engine.get_decoding_config()
709
710 - async def do_log_stats(self) -> None:
710 + async def do_log_stats(
711 + self,
712 + scheduler_outputs: Optional[SchedulerOutputs] = None,
713 + model_output: Optional[List[SamplerOutput]] = None) -> None:
714 if self.engine_use_ray:
712 - await self.engine.do_log_stats.remote() # type: ignore
715 + await self.engine.do_log_stats.remote( # type: ignore
716 + scheduler_outputs, model_output)
717 else:
718 self.engine.do_log_stats()
719
vllm/engine/llm_engine.py
+7 -5
@@ -597,16 +597,18 @@ class LLMEngine:
597 scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
598
599 # Log stats.
600 - if self.log_stats:
601 - self.stat_logger.log(
602 - self._get_stats(scheduler_outputs, model_output=output))
600 + self.do_log_stats(scheduler_outputs, output)
601
602 return request_outputs
603
606 - def do_log_stats(self) -> None:
604 + def do_log_stats(
605 + self,
606 + scheduler_outputs: Optional[SchedulerOutputs] = None,
607 + model_output: Optional[List[SamplerOutput]] = None) -> None:
608 """Forced log when no requests active."""
609 if self.log_stats:
609 - self.stat_logger.log(self._get_stats(scheduler_outputs=None))
610 + self.stat_logger.log(
611 + self._get_stats(scheduler_outputs, model_output))
612
613 def _get_stats(
614 self,