A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Miscalculated latency lead to time_to_first_token_seconds inaccurate. (#6686)
Allen.Dou committed
Jul 24, 2024 at 23:58 UTC
40468b13faa1ebde366e7002c5752b59e1368d10
2 files changed
+3
-2
vllm/engine/llm_engine.py
+2
-1
@@ -949,8 +949,9 @@ class LLMEngine:
949
model_output: Optional[List[SamplerOutput]] = None) -> None:
950
"""Forced log when no requests active."""
951
if self.log_stats:
952
+ stats = self._get_stats(scheduler_outputs, model_output)
953
for logger in self.stat_loggers.values():
953
- logger.log(self._get_stats(scheduler_outputs, model_output))
954
+ logger.log(stats)
955
956
def _get_stats(
957
self,
vllm/spec_decode/spec_decode_worker.py
+1
-1
@@ -484,7 +484,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
484
for both speculation cases (num_lookahead_slots>0) and non-speculation
485
cases (e.g. prefill).
486
487
- Returns True iff there are remaining sequences to process.
487
+ Returns True if there are remaining sequences to process.
488
"""
489
assert self.rank != self._driver_rank
490