@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7be4f5628f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix][Core] Restore logging of stats in the async engine (#4150)

Ronen Schaffer committed Apr 19, 2024 at 18:08 UTC 7be4f5628fc9999bf8a6025edd8f098353e0724b
1 file changed +7 -1
vllm/engine/async_llm_engine.py
+7 -1
@@ -217,10 +217,16 @@ class _AsyncLLMEngine(LLMEngine):
217 else:
218 output = []
219
220 - return self._process_model_outputs(
220 + request_outputs = self._process_model_outputs(
221 output, scheduler_outputs.scheduled_seq_groups,
222 scheduler_outputs.ignored_seq_groups)
223
224 + # Log stats.
225 + if self.log_stats:
226 + self.stat_logger.log(self._get_stats(scheduler_outputs))
227 +
228 + return request_outputs
229 +
230 async def encode_request_async(
231 self,
232 request_id: str, # pylint: disable=unused-argument