@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 017d9f1515
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add metrics to RequestOutput (#2876)

Antoni Baum committed Feb 20, 2024 at 21:55 UTC 017d9f15151ce571a5f4fd381699c72a872636ec
6 files changed +61 -9
tests/async_engine/test_request_tracker.py
+1 -1
@@ -64,7 +64,7 @@ def test_request_tracker():
64 stream_5 = tracker.add_request("5")
65 assert tracker.new_requests_event.flag
66 tracker.process_request_output(
67 - RequestOutput("2", "output", [], [], [], finished=True))
67 + RequestOutput("2", "output", [], [], [], bool(finished)))
68 new, finished = tracker.get_new_and_finished_requests()
69 assert not tracker.new_requests_event.flag
70 assert len(finished) == 1
vllm/core/policy.py
+1 -1
@@ -33,7 +33,7 @@ class FCFS(Policy):
33 now: float,
34 seq_group: SequenceGroup,
35 ) -> float:
36 - return now - seq_group.arrival_time
36 + return now - seq_group.metrics.arrival_time
37
38
39 class PolicyFactory:
vllm/core/scheduler.py
+3
@@ -365,10 +365,13 @@ class Scheduler:
365 # This function call changes the internal states of the scheduler
366 # such as self.running, self.swapped, and self.waiting.
367 scheduler_outputs = self._schedule()
368 + now = time.time()
369
370 # Create input data structures.
371 seq_group_metadata_list: List[SequenceGroupMetadata] = []
372 for seq_group in scheduler_outputs.scheduled_seq_groups:
373 + seq_group.maybe_set_first_scheduled_time(now)
374 +
375 seq_data: Dict[int, SequenceData] = {}
376 block_tables: Dict[int, List[int]] = {}
377 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
vllm/engine/llm_engine.py
+5 -2
@@ -728,6 +728,7 @@ class LLMEngine:
728 def _process_model_outputs(
729 self, output: SamplerOutput,
730 scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
731 + now = time.time()
732 # Update the scheduled sequence groups with the model outputs.
733 scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
734 for seq_group, outputs in zip(scheduled_seq_groups, output):
@@ -739,6 +740,7 @@ class LLMEngine:
740 # Create the outputs.
741 request_outputs: List[RequestOutput] = []
742 for seq_group in scheduled_seq_groups:
743 + seq_group.maybe_set_first_token_time(now)
744 request_output = RequestOutput.from_seq_group(seq_group)
745 request_outputs.append(request_output)
746 for seq_group in scheduler_outputs.ignored_seq_groups:
@@ -876,11 +878,12 @@ class LLMEngine:
878 # Latency Timings.
879 time_last_iters = []
880 for seq_group in scheduler_outputs.scheduled_seq_groups:
879 - # Time since last token. (n.b. updates seq_group.last_token_time)
881 + # Time since last token. (n.b. updates seq_group.metrics.last_token_time)
882 time_last_iters.append(seq_group.get_last_latency(now))
883 # Time since arrival for all finished requests.
884 if seq_group.is_finished():
883 - time_e2e_requests.append(now - seq_group.arrival_time)
885 + time_e2e_requests.append(now -
886 + seq_group.metrics.arrival_time)
887
888 time_to_first_tokens = time_last_iters if prompt_run else []
889 time_per_output_tokens = [] if prompt_run else time_last_iters
vllm/outputs.py
+9 -1
@@ -1,7 +1,8 @@
1 from typing import List, Optional
2 +import time
3
4 from vllm.sequence import (PromptLogprobs, SampleLogprobs, SequenceGroup,
4 - SequenceStatus)
5 + SequenceStatus, RequestMetrics)
6 from vllm.lora.request import LoRARequest
7
8
@@ -60,6 +61,7 @@ class RequestOutput:
61 prompt_logprobs: The log probabilities to return per prompt token.
62 outputs: The output sequences of the request.
63 finished: Whether the whole request is finished.
64 + metrics: Metrics associated with the request.
65 lora_request: The LoRA request that was used to generate the output.
66 """
67
@@ -71,6 +73,7 @@ class RequestOutput:
73 prompt_logprobs: Optional[PromptLogprobs],
74 outputs: List[CompletionOutput],
75 finished: bool,
76 + metrics: Optional[RequestMetrics] = None,
77 lora_request: Optional[LoRARequest] = None,
78 ) -> None:
79 self.request_id = request_id
@@ -79,6 +82,7 @@ class RequestOutput:
82 self.prompt_logprobs = prompt_logprobs
83 self.outputs = outputs
84 self.finished = finished
85 + self.metrics = metrics
86 self.lora_request = lora_request
87
88 @classmethod
@@ -115,12 +119,15 @@ class RequestOutput:
119 prompt_token_ids = seq_group.prompt_token_ids
120 prompt_logprobs = seq_group.prompt_logprobs
121 finished = seq_group.is_finished()
122 + finished_time = time.time() if finished else None
123 + seq_group.set_finished_time(finished_time)
124 return cls(seq_group.request_id,
125 prompt,
126 prompt_token_ids,
127 prompt_logprobs,
128 outputs,
129 finished,
130 + seq_group.metrics,
131 lora_request=seq_group.lora_request)
132
133 def __repr__(self) -> str:
@@ -130,4 +137,5 @@ class RequestOutput:
137 f"prompt_logprobs={self.prompt_logprobs}, "
138 f"outputs={self.outputs}, "
139 f"finished={self.finished}, "
140 + f"metrics={self.metrics}, "
141 f"lora_request={self.lora_request})")
vllm/sequence.py
+42 -4
@@ -1,6 +1,7 @@
1 """Sequence and its related classes."""
2 import copy
3 import enum
4 +from dataclasses import dataclass
5 from typing import Dict, List, Optional, Union
6
7 from vllm.block import LogicalTokenBlock
@@ -49,6 +50,25 @@ class SequenceStatus(enum.Enum):
50 return finish_reason
51
52
53 +@dataclass
54 +class RequestMetrics:
55 + """Metrics associated with a request.
56 +
57 + Args:
58 + arrival_time: The time when the request arrived.
59 + first_scheduled_time: The time when the request was first scheduled.
60 + first_token_time: The time when the first token was generated.
61 + time_in_queue: The time the request spent in the queue.
62 + finished_time: The time when the request was finished.
63 + """
64 + arrival_time: float
65 + last_token_time: float
66 + first_scheduled_time: Optional[float]
67 + first_token_time: Optional[float]
68 + time_in_queue: Optional[float]
69 + finished_time: Optional[float] = None
70 +
71 +
72 class SequenceData:
73 """Data associated with a sequence.
74
@@ -252,8 +272,11 @@ class SequenceGroup:
272 self.request_id = request_id
273 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
274 self.sampling_params = sampling_params
255 - self.arrival_time = arrival_time
256 - self.last_token_time = arrival_time
275 + self.metrics = RequestMetrics(arrival_time=arrival_time,
276 + last_token_time=arrival_time,
277 + first_scheduled_time=None,
278 + first_token_time=None,
279 + time_in_queue=None)
280 self.lora_request = lora_request
281 self.prefix: Optional[Prefix] = prefix
282 self.prompt_logprobs: Optional[PromptLogprobs] = None
@@ -276,10 +299,25 @@ class SequenceGroup:
299
300 def get_last_latency(self, now: float) -> float:
301 """Gets last token latency for Request level timings."""
279 - latency = now - self.last_token_time
280 - self.last_token_time = now
302 + latency = now - self.metrics.last_token_time
303 + self.metrics.last_token_time = now
304 return latency
305
306 + def maybe_set_first_token_time(self, time: float) -> None:
307 + """Sets the first token time for Request level timings."""
308 + if self.metrics.first_token_time is None:
309 + self.metrics.first_token_time = time
310 +
311 + def maybe_set_first_scheduled_time(self, time: float) -> None:
312 + """Sets the first scheduled time and time in queue for Request level timings."""
313 + if self.metrics.first_scheduled_time is None:
314 + self.metrics.first_scheduled_time = time
315 + self.metrics.time_in_queue = time - self.metrics.arrival_time
316 +
317 + def set_finished_time(self, time: Optional[float]) -> None:
318 + """Sets the finished time for Request level timings."""
319 + self.metrics.finished_time = time
320 +
321 def get_max_num_running_seqs(self) -> int:
322 """The maximum number of sequences running in parallel in the remaining
323 lifetime of the request."""