A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add metrics to RequestOutput (#2876)
Antoni Baum committed
Feb 20, 2024 at 21:55 UTC
017d9f15151ce571a5f4fd381699c72a872636ec
6 files changed
+61
-9
tests/async_engine/test_request_tracker.py
+1
-1
@@ -64,7 +64,7 @@ def test_request_tracker():
64
stream_5 = tracker.add_request("5")
65
assert tracker.new_requests_event.flag
66
tracker.process_request_output(
67
- RequestOutput("2", "output", [], [], [], finished=True))
67
+ RequestOutput("2", "output", [], [], [], bool(finished)))
68
new, finished = tracker.get_new_and_finished_requests()
69
assert not tracker.new_requests_event.flag
70
assert len(finished) == 1
vllm/core/policy.py
+1
-1
@@ -33,7 +33,7 @@ class FCFS(Policy):
33
now: float,
34
seq_group: SequenceGroup,
35
) -> float:
36
- return now - seq_group.arrival_time
36
+ return now - seq_group.metrics.arrival_time
37
38
39
class PolicyFactory:
vllm/core/scheduler.py
+3
@@ -365,10 +365,13 @@ class Scheduler:
365
# This function call changes the internal states of the scheduler
366
# such as self.running, self.swapped, and self.waiting.
367
scheduler_outputs = self._schedule()
368
+ now = time.time()
369
370
# Create input data structures.
371
seq_group_metadata_list: List[SequenceGroupMetadata] = []
372
for seq_group in scheduler_outputs.scheduled_seq_groups:
373
+ seq_group.maybe_set_first_scheduled_time(now)
374
+
375
seq_data: Dict[int, SequenceData] = {}
376
block_tables: Dict[int, List[int]] = {}
377
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
vllm/engine/llm_engine.py
+5
-2
@@ -728,6 +728,7 @@ class LLMEngine:
728
def _process_model_outputs(
729
self, output: SamplerOutput,
730
scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
731
+ now = time.time()
732
# Update the scheduled sequence groups with the model outputs.
733
scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
734
for seq_group, outputs in zip(scheduled_seq_groups, output):
@@ -739,6 +740,7 @@ class LLMEngine:
740
# Create the outputs.
741
request_outputs: List[RequestOutput] = []
742
for seq_group in scheduled_seq_groups:
743
+ seq_group.maybe_set_first_token_time(now)
744
request_output = RequestOutput.from_seq_group(seq_group)
745
request_outputs.append(request_output)
746
for seq_group in scheduler_outputs.ignored_seq_groups:
@@ -876,11 +878,12 @@ class LLMEngine:
878
# Latency Timings.
879
time_last_iters = []
880
for seq_group in scheduler_outputs.scheduled_seq_groups:
879
- # Time since last token. (n.b. updates seq_group.last_token_time)
881
+ # Time since last token. (n.b. updates seq_group.metrics.last_token_time)
882
time_last_iters.append(seq_group.get_last_latency(now))
883
# Time since arrival for all finished requests.
884
if seq_group.is_finished():
883
- time_e2e_requests.append(now - seq_group.arrival_time)
885
+ time_e2e_requests.append(now -
886
+ seq_group.metrics.arrival_time)
887
888
time_to_first_tokens = time_last_iters if prompt_run else []
889
time_per_output_tokens = [] if prompt_run else time_last_iters
vllm/outputs.py
+9
-1
@@ -1,7 +1,8 @@
1
from typing import List, Optional
2
+import time
3
4
from vllm.sequence import (PromptLogprobs, SampleLogprobs, SequenceGroup,
4
- SequenceStatus)
5
+ SequenceStatus, RequestMetrics)
6
from vllm.lora.request import LoRARequest
7
8
@@ -60,6 +61,7 @@ class RequestOutput:
61
prompt_logprobs: The log probabilities to return per prompt token.
62
outputs: The output sequences of the request.
63
finished: Whether the whole request is finished.
64
+ metrics: Metrics associated with the request.
65
lora_request: The LoRA request that was used to generate the output.
66
"""
67
@@ -71,6 +73,7 @@ class RequestOutput:
73
prompt_logprobs: Optional[PromptLogprobs],
74
outputs: List[CompletionOutput],
75
finished: bool,
76
+ metrics: Optional[RequestMetrics] = None,
77
lora_request: Optional[LoRARequest] = None,
78
) -> None:
79
self.request_id = request_id
@@ -79,6 +82,7 @@ class RequestOutput:
82
self.prompt_logprobs = prompt_logprobs
83
self.outputs = outputs
84
self.finished = finished
85
+ self.metrics = metrics
86
self.lora_request = lora_request
87
88
@classmethod
@@ -115,12 +119,15 @@ class RequestOutput:
119
prompt_token_ids = seq_group.prompt_token_ids
120
prompt_logprobs = seq_group.prompt_logprobs
121
finished = seq_group.is_finished()
122
+ finished_time = time.time() if finished else None
123
+ seq_group.set_finished_time(finished_time)
124
return cls(seq_group.request_id,
125
prompt,
126
prompt_token_ids,
127
prompt_logprobs,
128
outputs,
129
finished,
130
+ seq_group.metrics,
131
lora_request=seq_group.lora_request)
132
133
def __repr__(self) -> str:
@@ -130,4 +137,5 @@ class RequestOutput:
137
f"prompt_logprobs={self.prompt_logprobs}, "
138
f"outputs={self.outputs}, "
139
f"finished={self.finished}, "
140
+ f"metrics={self.metrics}, "
141
f"lora_request={self.lora_request})")
vllm/sequence.py
+42
-4
@@ -1,6 +1,7 @@
1
"""Sequence and its related classes."""
2
import copy
3
import enum
4
+from dataclasses import dataclass
5
from typing import Dict, List, Optional, Union
6
7
from vllm.block import LogicalTokenBlock
@@ -49,6 +50,25 @@ class SequenceStatus(enum.Enum):
50
return finish_reason
51
52
53
+@dataclass
54
+class RequestMetrics:
55
+ """Metrics associated with a request.
56
+
57
+ Args:
58
+ arrival_time: The time when the request arrived.
59
+ first_scheduled_time: The time when the request was first scheduled.
60
+ first_token_time: The time when the first token was generated.
61
+ time_in_queue: The time the request spent in the queue.
62
+ finished_time: The time when the request was finished.
63
+ """
64
+ arrival_time: float
65
+ last_token_time: float
66
+ first_scheduled_time: Optional[float]
67
+ first_token_time: Optional[float]
68
+ time_in_queue: Optional[float]
69
+ finished_time: Optional[float] = None
70
+
71
+
72
class SequenceData:
73
"""Data associated with a sequence.
74
@@ -252,8 +272,11 @@ class SequenceGroup:
272
self.request_id = request_id
273
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
274
self.sampling_params = sampling_params
255
- self.arrival_time = arrival_time
256
- self.last_token_time = arrival_time
275
+ self.metrics = RequestMetrics(arrival_time=arrival_time,
276
+ last_token_time=arrival_time,
277
+ first_scheduled_time=None,
278
+ first_token_time=None,
279
+ time_in_queue=None)
280
self.lora_request = lora_request
281
self.prefix: Optional[Prefix] = prefix
282
self.prompt_logprobs: Optional[PromptLogprobs] = None
@@ -276,10 +299,25 @@ class SequenceGroup:
299
300
def get_last_latency(self, now: float) -> float:
301
"""Gets last token latency for Request level timings."""
279
- latency = now - self.last_token_time
280
- self.last_token_time = now
302
+ latency = now - self.metrics.last_token_time
303
+ self.metrics.last_token_time = now
304
return latency
305
306
+ def maybe_set_first_token_time(self, time: float) -> None:
307
+ """Sets the first token time for Request level timings."""
308
+ if self.metrics.first_token_time is None:
309
+ self.metrics.first_token_time = time
310
+
311
+ def maybe_set_first_scheduled_time(self, time: float) -> None:
312
+ """Sets the first scheduled time and time in queue for Request level timings."""
313
+ if self.metrics.first_scheduled_time is None:
314
+ self.metrics.first_scheduled_time = time
315
+ self.metrics.time_in_queue = time - self.metrics.arrival_time
316
+
317
+ def set_finished_time(self, time: Optional[float]) -> None:
318
+ """Sets the finished time for Request level timings."""
319
+ self.metrics.finished_time = time
320
+
321
def get_max_num_running_seqs(self) -> int:
322
"""The maximum number of sequences running in parallel in the remaining
323
lifetime of the request."""