A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Last token measurement fix (#11376)
Signed-off-by: rajveerb <46040700+rajveerb@users.noreply.github.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Rajveer Bachkaniwala committed
Dec 27, 2024 at 22:34 UTC
b5cbe8eeb30e86c8477d91c66f5c7a10e4ee754b
2 files changed
+20
-12
vllm/engine/llm_engine.py
+6
-2
@@ -1124,6 +1124,8 @@ class LLMEngine:
1124
1125
seq_group = scheduled_seq_group.seq_group
1126
seq_group.maybe_set_first_token_time(now)
1127
+ if not seq_group.is_prefill():
1128
+ seq_group.set_last_token_time(now)
1129
request_output = RequestOutputFactory.create(
1130
seq_group,
1131
self.seq_id_to_seq_group,
@@ -1166,6 +1168,8 @@ class LLMEngine:
1168
1169
seq_group = scheduled_seq_group.seq_group
1170
seq_group.maybe_set_first_token_time(now)
1171
+ if not seq_group.is_prefill():
1172
+ seq_group.set_last_token_time(now)
1173
request_output = RequestOutputFactory.create(
1174
seq_group,
1175
self.seq_id_to_seq_group,
@@ -1686,7 +1690,7 @@ class LLMEngine:
1690
# If the seq_group just finished the prefill state
1691
# get TTFT.
1692
if not seq_group.is_prefill():
1689
- latency = seq_group.get_last_latency(now)
1693
+ latency = seq_group.get_last_token_latency()
1694
time_to_first_tokens_iter.append(latency)
1695
1696
# One generation token per finished prefill.
@@ -1694,7 +1698,7 @@ class LLMEngine:
1698
seq_group.num_seqs())
1699
else:
1700
# TPOTs.
1697
- latency = seq_group.get_last_latency(now)
1701
+ latency = seq_group.get_last_token_latency()
1702
time_per_output_tokens_iter.append(latency)
1703
if seq_group.state.current_step == 0:
1704
# For async_output_proc, the do_log_stats()
vllm/sequence.py
+14
-10
@@ -667,6 +667,7 @@ class SequenceGroup:
667
first_scheduled_time=None,
668
first_token_time=None,
669
time_in_queue=None)
670
+ self.last_token_latency = 0.0
671
self.lora_request = lora_request
672
self.prompt_logprobs: Optional[PromptLogprobs] = None
673
self.state = SequenceGroupState()
@@ -762,18 +763,21 @@ class SequenceGroup:
763
assert num_lookahead_slots + 1 == num_scheduler_steps or is_prefill
764
self.init_multi_step(num_steps=num_lookahead_slots + 1)
765
765
- def get_last_latency(self, now: float) -> float:
766
+ def set_last_token_time(self, now: float) -> None:
767
"""Sets the last token time for Request level timings."""
767
- # If still in prefill phase, raise Error.
768
- if self.is_prefill():
769
- raise ValueError(
770
- "seq_group.get_last_latency() should not be called "
771
- "if the seq_group is in prefill phase.")
772
-
773
- # Otherwise return token latency.
774
- latency = now - self.metrics.last_token_time
768
+ # If still in prefill phase, assertion fails.
769
+ assert not self.is_prefill(), (
770
+ "seq_group.set_last_token_time() should not be called "
771
+ "if the seq_group is in prefill phase.")
772
+ self.last_token_latency = now - self.metrics.last_token_time
773
self.metrics.last_token_time = now
776
- return latency
774
+
775
+ def get_last_token_latency(self) -> float:
776
+ """Returns the latency of the last token."""
777
+ assert not self.is_prefill(), (
778
+ "seq_group.get_last_token_latency() should not be called "
779
+ "if the seq_group is in prefill phase.")
780
+ return self.last_token_latency
781
782
def maybe_set_first_token_time(self, time: float) -> None:
783
"""Sets the first token time for Request level timings."""