@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 40468b13fa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Miscalculated latency lead to time_to_first_token_seconds inaccurate. (#6686)

Allen.Dou committed Jul 24, 2024 at 23:58 UTC 40468b13faa1ebde366e7002c5752b59e1368d10
2 files changed +3 -2
vllm/engine/llm_engine.py
+2 -1
@@ -949,8 +949,9 @@ class LLMEngine:
949 model_output: Optional[List[SamplerOutput]] = None) -> None:
950 """Forced log when no requests active."""
951 if self.log_stats:
952 + stats = self._get_stats(scheduler_outputs, model_output)
953 for logger in self.stat_loggers.values():
953 - logger.log(self._get_stats(scheduler_outputs, model_output))
954 + logger.log(stats)
955
956 def _get_stats(
957 self,
vllm/spec_decode/spec_decode_worker.py
+1 -1
@@ -484,7 +484,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
484 for both speculation cases (num_lookahead_slots>0) and non-speculation
485 cases (e.g. prefill).
486
487 - Returns True iff there are remaining sequences to process.
487 + Returns True if there are remaining sequences to process.
488 """
489 assert self.rank != self._driver_rank
490