A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix][Core] Restore logging of stats in the async engine (#4150)
Ronen Schaffer committed
Apr 19, 2024 at 18:08 UTC
7be4f5628fc9999bf8a6025edd8f098353e0724b
1 file changed
+7
-1
vllm/engine/async_llm_engine.py
+7
-1
@@ -217,10 +217,16 @@ class _AsyncLLMEngine(LLMEngine):
217
else:
218
output = []
219
220
- return self._process_model_outputs(
220
+ request_outputs = self._process_model_outputs(
221
output, scheduler_outputs.scheduled_seq_groups,
222
scheduler_outputs.ignored_seq_groups)
223
224
+ # Log stats.
225
+ if self.log_stats:
226
+ self.stat_logger.log(self._get_stats(scheduler_outputs))
227
+
228
+ return request_outputs
229
+
230
async def encode_request_async(
231
self,
232
request_id: str, # pylint: disable=unused-argument