@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b5cbe8eeb3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Last token measurement fix (#11376)

Signed-off-by: rajveerb <46040700+rajveerb@users.noreply.github.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>

Rajveer Bachkaniwala committed Dec 27, 2024 at 22:34 UTC b5cbe8eeb30e86c8477d91c66f5c7a10e4ee754b
2 files changed +20 -12
vllm/engine/llm_engine.py
+6 -2
@@ -1124,6 +1124,8 @@ class LLMEngine:
1124
1125 seq_group = scheduled_seq_group.seq_group
1126 seq_group.maybe_set_first_token_time(now)
1127 + if not seq_group.is_prefill():
1128 + seq_group.set_last_token_time(now)
1129 request_output = RequestOutputFactory.create(
1130 seq_group,
1131 self.seq_id_to_seq_group,
@@ -1166,6 +1168,8 @@ class LLMEngine:
1168
1169 seq_group = scheduled_seq_group.seq_group
1170 seq_group.maybe_set_first_token_time(now)
1171 + if not seq_group.is_prefill():
1172 + seq_group.set_last_token_time(now)
1173 request_output = RequestOutputFactory.create(
1174 seq_group,
1175 self.seq_id_to_seq_group,
@@ -1686,7 +1690,7 @@ class LLMEngine:
1690 # If the seq_group just finished the prefill state
1691 # get TTFT.
1692 if not seq_group.is_prefill():
1689 - latency = seq_group.get_last_latency(now)
1693 + latency = seq_group.get_last_token_latency()
1694 time_to_first_tokens_iter.append(latency)
1695
1696 # One generation token per finished prefill.
@@ -1694,7 +1698,7 @@ class LLMEngine:
1698 seq_group.num_seqs())
1699 else:
1700 # TPOTs.
1697 - latency = seq_group.get_last_latency(now)
1701 + latency = seq_group.get_last_token_latency()
1702 time_per_output_tokens_iter.append(latency)
1703 if seq_group.state.current_step == 0:
1704 # For async_output_proc, the do_log_stats()
vllm/sequence.py
+14 -10
@@ -667,6 +667,7 @@ class SequenceGroup:
667 first_scheduled_time=None,
668 first_token_time=None,
669 time_in_queue=None)
670 + self.last_token_latency = 0.0
671 self.lora_request = lora_request
672 self.prompt_logprobs: Optional[PromptLogprobs] = None
673 self.state = SequenceGroupState()
@@ -762,18 +763,21 @@ class SequenceGroup:
763 assert num_lookahead_slots + 1 == num_scheduler_steps or is_prefill
764 self.init_multi_step(num_steps=num_lookahead_slots + 1)
765
765 - def get_last_latency(self, now: float) -> float:
766 + def set_last_token_time(self, now: float) -> None:
767 """Sets the last token time for Request level timings."""
767 - # If still in prefill phase, raise Error.
768 - if self.is_prefill():
769 - raise ValueError(
770 - "seq_group.get_last_latency() should not be called "
771 - "if the seq_group is in prefill phase.")
772 -
773 - # Otherwise return token latency.
774 - latency = now - self.metrics.last_token_time
768 + # If still in prefill phase, assertion fails.
769 + assert not self.is_prefill(), (
770 + "seq_group.set_last_token_time() should not be called "
771 + "if the seq_group is in prefill phase.")
772 + self.last_token_latency = now - self.metrics.last_token_time
773 self.metrics.last_token_time = now
776 - return latency
774 +
775 + def get_last_token_latency(self) -> float:
776 + """Returns the latency of the last token."""
777 + assert not self.is_prefill(), (
778 + "seq_group.get_last_token_latency() should not be called "
779 + "if the seq_group is in prefill phase.")
780 + return self.last_token_latency
781
782 def maybe_set_first_token_time(self, time: float) -> None:
783 """Sets the first token time for Request level timings."""